From 4eb61529f67ab124de16f78e0e4bb2e18fdee45d Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 12:35:04 +0200 Subject: [PATCH 01/68] */+ rename prog8Parser (generated java) to Prog8ANTLRParser; add Kotlin class Prog8Parser to interface with it --- .idea/compiler.xml | 1 + .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 109 +++++++++--------- .../parser/CommentHandlingTokenStream.kt | 2 +- compilerAst/src/prog8/parser/ModuleParsing.kt | 43 ++----- compilerAst/src/prog8/parser/Prog8Parser.kt | 65 +++++++++++ compilerAst/test/TestAntlrParser.kt | 31 +++-- parser/antlr/{prog8.g4 => Prog8ANTLR.g4} | 4 +- 7 files changed, 146 insertions(+), 109 deletions(-) create mode 100644 compilerAst/src/prog8/parser/Prog8Parser.kt rename parser/antlr/{prog8.g4 => Prog8ANTLR.g4} (97%) diff --git a/.idea/compiler.xml b/.idea/compiler.xml index a39e0f0fb..af8a35b20 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -2,5 +2,6 @@ \ No newline at end of file diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index fc07eece1..0cf6e7eac 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -8,8 +8,7 @@ import prog8.ast.Module import prog8.ast.base.* import prog8.ast.expressions.* import prog8.ast.statements.* -import prog8.parser.CustomLexer -import prog8.parser.prog8Parser +import prog8.parser.Prog8ANTLRParser import java.io.CharConversionException import java.io.File import java.nio.file.Path @@ -19,7 +18,7 @@ import java.nio.file.Path private data class NumericLiteral(val number: Number, val datatype: DataType) -internal fun prog8Parser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { +internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name val directives = this.directive().map { it.toAst() } val blocks = this.block().map { it.toAst(Module.isLibrary(source), encoding) } @@ -27,6 +26,7 @@ internal fun prog8Parser.ModuleContext.toAst(name: String, source: Path, encodin } private fun ParserRuleContext.toPosition() : Position { + /* val customTokensource = this.start.tokenSource as? CustomLexer val filename = when { @@ -34,11 +34,14 @@ private fun ParserRuleContext.toPosition() : Position { start.tokenSource.sourceName == IntStream.UNKNOWN_SOURCE_NAME -> "@internal@" else -> File(start.inputStream.sourceName).name } + */ + val filename = start.inputStream.sourceName + // note: be ware of TAB characters in the source text, they count as 1 column... return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine + stop.text.length) } -private fun prog8Parser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { val blockstatements = block_statement().map { when { it.variabledeclaration()!=null -> it.variabledeclaration().toAst(encoding) @@ -52,10 +55,10 @@ private fun prog8Parser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStri return Block(identifier().text, integerliteral()?.toAst()?.number?.toInt(), blockstatements.toMutableList(), isInLibrary, toPosition()) } -private fun prog8Parser.Statement_blockContext.toAst(encoding: IStringEncoding): MutableList = +private fun Prog8ANTLRParser.Statement_blockContext.toAst(encoding: IStringEncoding): MutableList = statement().asSequence().map { it.toAst(encoding) }.toMutableList() -private fun prog8Parser.VariabledeclarationContext.toAst(encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringEncoding) : Statement { vardecl()?.let { return it.toAst(encoding) } varinitializer()?.let { @@ -111,7 +114,7 @@ private fun prog8Parser.VariabledeclarationContext.toAst(encoding: IStringEncodi throw FatalAstException("weird variable decl $this") } -private fun prog8Parser.SubroutinedeclarationContext.toAst(encoding: IStringEncoding) : Subroutine { +private fun Prog8ANTLRParser.SubroutinedeclarationContext.toAst(encoding: IStringEncoding) : Subroutine { return when { subroutine()!=null -> subroutine().toAst(encoding) asmsubroutine()!=null -> asmsubroutine().toAst(encoding) @@ -120,7 +123,7 @@ private fun prog8Parser.SubroutinedeclarationContext.toAst(encoding: IStringEnco } } -private fun prog8Parser.StatementContext.toAst(encoding: IStringEncoding) : Statement { +private fun Prog8ANTLRParser.StatementContext.toAst(encoding: IStringEncoding) : Statement { val vardecl = variabledeclaration()?.toAst(encoding) if(vardecl!=null) return vardecl @@ -188,7 +191,7 @@ private fun prog8Parser.StatementContext.toAst(encoding: IStringEncoding) : Stat throw FatalAstException("unprocessed source text (are we missing ast conversion rules for parser elements?): $text") } -private fun prog8Parser.AsmsubroutineContext.toAst(encoding: IStringEncoding): Subroutine { +private fun Prog8ANTLRParser.AsmsubroutineContext.toAst(encoding: IStringEncoding): Subroutine { val inline = this.inline()!=null val subdecl = asmsub_decl().toAst() val statements = statement_block()?.toAst(encoding) ?: mutableListOf() @@ -197,7 +200,7 @@ private fun prog8Parser.AsmsubroutineContext.toAst(encoding: IStringEncoding): S subdecl.asmClobbers, null, true, inline, statements, toPosition()) } -private fun prog8Parser.RomsubroutineContext.toAst(): Subroutine { +private fun Prog8ANTLRParser.RomsubroutineContext.toAst(): Subroutine { val subdecl = asmsub_decl().toAst() val address = integerliteral().toAst().number.toInt() return Subroutine(subdecl.name, subdecl.parameters, subdecl.returntypes, @@ -213,7 +216,7 @@ private class AsmsubDecl(val name: String, val asmReturnvaluesRegisters: List, val asmClobbers: Set) -private fun prog8Parser.Asmsub_declContext.toAst(): AsmsubDecl { +private fun Prog8ANTLRParser.Asmsub_declContext.toAst(): AsmsubDecl { val name = identifier().text val params = asmsub_params()?.toAst() ?: emptyList() val returns = asmsub_returns()?.toAst() ?: emptyList() @@ -236,7 +239,7 @@ private class AsmSubroutineReturn(val type: DataType, val statusflag: Statusflag?, val position: Position) -private fun prog8Parser.Asmsub_returnsContext.toAst(): List +private fun Prog8ANTLRParser.Asmsub_returnsContext.toAst(): List = asmsub_return().map { val register = it.register().text var registerorpair: RegisterOrPair? = null @@ -255,7 +258,7 @@ private fun prog8Parser.Asmsub_returnsContext.toAst(): List toPosition()) } -private fun prog8Parser.Asmsub_paramsContext.toAst(): List +private fun Prog8ANTLRParser.Asmsub_paramsContext.toAst(): List = asmsub_param().map { val vardecl = it.vardecl() val datatype = vardecl.datatype()?.toAst() ?: DataType.UNDEFINED @@ -272,7 +275,7 @@ private fun prog8Parser.Asmsub_paramsContext.toAst(): List { +private fun Prog8ANTLRParser.Sub_return_partContext.toAst(): List { val returns = sub_returns() ?: return emptyList() return returns.datatype().map { it.toAst() } } -private fun prog8Parser.Sub_paramsContext.toAst(): List = +private fun Prog8ANTLRParser.Sub_paramsContext.toAst(): List = vardecl().map { val datatype = it.datatype()?.toAst() ?: DataType.UNDEFINED SubroutineParameter(it.varname.text, datatype, it.toPosition()) } -private fun prog8Parser.Assign_targetContext.toAst(encoding: IStringEncoding) : AssignTarget { +private fun Prog8ANTLRParser.Assign_targetContext.toAst(encoding: IStringEncoding) : AssignTarget { val identifier = scoped_identifier() return when { identifier!=null -> AssignTarget(identifier.toAst(), null, null, toPosition()) @@ -338,20 +341,20 @@ private fun prog8Parser.Assign_targetContext.toAst(encoding: IStringEncoding) : } } -private fun prog8Parser.ClobberContext.toAst() : Set { +private fun Prog8ANTLRParser.ClobberContext.toAst() : Set { val names = this.cpuregister().map { it.text } return names.map { CpuRegister.valueOf(it) }.toSet() } -private fun prog8Parser.DatatypeContext.toAst() = DataType.valueOf(text.uppercase()) +private fun Prog8ANTLRParser.DatatypeContext.toAst() = DataType.valueOf(text.uppercase()) -private fun prog8Parser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = +private fun Prog8ANTLRParser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = ArrayIndex(expression().toAst(encoding), toPosition()) -private fun prog8Parser.DirectiveContext.toAst() : Directive = +private fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = Directive(directivename.text, directivearg().map { it.toAst() }, toPosition()) -private fun prog8Parser.DirectiveargContext.toAst() : DirectiveArg { +private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { val str = stringliteral() if(str?.ALT_STRING_ENCODING() != null) throw AstException("${toPosition()} can't use alternate string encodings for directive arguments") @@ -359,7 +362,7 @@ private fun prog8Parser.DirectiveargContext.toAst() : DirectiveArg { return DirectiveArg(stringliteral()?.text, identifier()?.text, integerliteral()?.toAst()?.number?.toInt(), toPosition()) } -private fun prog8Parser.IntegerliteralContext.toAst(): NumericLiteral { +private fun Prog8ANTLRParser.IntegerliteralContext.toAst(): NumericLiteral { fun makeLiteral(text: String, radix: Int): NumericLiteral { val integer: Int var datatype = DataType.UBYTE @@ -403,14 +406,14 @@ private fun prog8Parser.IntegerliteralContext.toAst(): NumericLiteral { val terminal: TerminalNode = children[0] as TerminalNode val integerPart = this.intpart.text return when (terminal.symbol.type) { - prog8Parser.DEC_INTEGER -> makeLiteral(integerPart, 10) - prog8Parser.HEX_INTEGER -> makeLiteral(integerPart.substring(1), 16) - prog8Parser.BIN_INTEGER -> makeLiteral(integerPart.substring(1), 2) + Prog8ANTLRParser.DEC_INTEGER -> makeLiteral(integerPart, 10) + Prog8ANTLRParser.HEX_INTEGER -> makeLiteral(integerPart.substring(1), 16) + Prog8ANTLRParser.BIN_INTEGER -> makeLiteral(integerPart.substring(1), 2) else -> throw FatalAstException(terminal.text) } } -private fun prog8Parser.ExpressionContext.toAst(encoding: IStringEncoding) : Expression { +private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) : Expression { val litval = literalvalue() if(litval!=null) { @@ -487,35 +490,35 @@ private fun prog8Parser.ExpressionContext.toAst(encoding: IStringEncoding) : Exp throw FatalAstException(text) } -private fun prog8Parser.StringliteralContext.toAst(): StringLiteralValue = +private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue = StringLiteralValue(unescape(this.STRING().text, toPosition()), ALT_STRING_ENCODING()!=null, toPosition()) -private fun prog8Parser.ArrayindexedContext.toAst(encoding: IStringEncoding): ArrayIndexedExpression { +private fun Prog8ANTLRParser.ArrayindexedContext.toAst(encoding: IStringEncoding): ArrayIndexedExpression { return ArrayIndexedExpression(scoped_identifier().toAst(), arrayindex().toAst(encoding), toPosition()) } -private fun prog8Parser.Expression_listContext.toAst(encoding: IStringEncoding) = expression().map{ it.toAst(encoding) } +private fun Prog8ANTLRParser.Expression_listContext.toAst(encoding: IStringEncoding) = expression().map{ it.toAst(encoding) } -private fun prog8Parser.IdentifierContext.toAst() : IdentifierReference = +private fun Prog8ANTLRParser.IdentifierContext.toAst() : IdentifierReference = IdentifierReference(listOf(text), toPosition()) -private fun prog8Parser.Scoped_identifierContext.toAst() : IdentifierReference = +private fun Prog8ANTLRParser.Scoped_identifierContext.toAst() : IdentifierReference = IdentifierReference(NAME().map { it.text }, toPosition()) -private fun prog8Parser.FloatliteralContext.toAst() = text.toDouble() +private fun Prog8ANTLRParser.FloatliteralContext.toAst() = text.toDouble() -private fun prog8Parser.BooleanliteralContext.toAst() = when(text) { +private fun Prog8ANTLRParser.BooleanliteralContext.toAst() = when(text) { "true" -> true "false" -> false else -> throw FatalAstException(text) } -private fun prog8Parser.ArrayliteralContext.toAst(encoding: IStringEncoding) : Array = +private fun Prog8ANTLRParser.ArrayliteralContext.toAst(encoding: IStringEncoding) : Array = expression().map { it.toAst(encoding) }.toTypedArray() -private fun prog8Parser.If_stmtContext.toAst(encoding: IStringEncoding): IfStatement { +private fun Prog8ANTLRParser.If_stmtContext.toAst(encoding: IStringEncoding): IfStatement { val condition = expression().toAst(encoding) val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() @@ -525,11 +528,11 @@ private fun prog8Parser.If_stmtContext.toAst(encoding: IStringEncoding): IfState return IfStatement(condition, trueScope, elseScope, toPosition()) } -private fun prog8Parser.Else_partContext.toAst(encoding: IStringEncoding): MutableList { +private fun Prog8ANTLRParser.Else_partContext.toAst(encoding: IStringEncoding): MutableList { return statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) } -private fun prog8Parser.Branch_stmtContext.toAst(encoding: IStringEncoding): BranchStatement { +private fun Prog8ANTLRParser.Branch_stmtContext.toAst(encoding: IStringEncoding): BranchStatement { val branchcondition = branchcondition().toAst() val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() @@ -539,11 +542,11 @@ private fun prog8Parser.Branch_stmtContext.toAst(encoding: IStringEncoding): Bra return BranchStatement(branchcondition, trueScope, elseScope, toPosition()) } -private fun prog8Parser.BranchconditionContext.toAst() = BranchCondition.valueOf( +private fun Prog8ANTLRParser.BranchconditionContext.toAst() = BranchCondition.valueOf( text.substringAfter('_').uppercase() ) -private fun prog8Parser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop { +private fun Prog8ANTLRParser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop { val loopvar = identifier().toAst() val iterable = expression()!!.toAst(encoding) val scope = @@ -554,9 +557,9 @@ private fun prog8Parser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop return ForLoop(loopvar, iterable, scope, toPosition()) } -private fun prog8Parser.BreakstmtContext.toAst() = Break(toPosition()) +private fun Prog8ANTLRParser.BreakstmtContext.toAst() = Break(toPosition()) -private fun prog8Parser.WhileloopContext.toAst(encoding: IStringEncoding): WhileLoop { +private fun Prog8ANTLRParser.WhileloopContext.toAst(encoding: IStringEncoding): WhileLoop { val condition = expression().toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -564,7 +567,7 @@ private fun prog8Parser.WhileloopContext.toAst(encoding: IStringEncoding): While return WhileLoop(condition, scope, toPosition()) } -private fun prog8Parser.RepeatloopContext.toAst(encoding: IStringEncoding): RepeatLoop { +private fun Prog8ANTLRParser.RepeatloopContext.toAst(encoding: IStringEncoding): RepeatLoop { val iterations = expression()?.toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -572,7 +575,7 @@ private fun prog8Parser.RepeatloopContext.toAst(encoding: IStringEncoding): Repe return RepeatLoop(iterations, scope, toPosition()) } -private fun prog8Parser.UntilloopContext.toAst(encoding: IStringEncoding): UntilLoop { +private fun Prog8ANTLRParser.UntilloopContext.toAst(encoding: IStringEncoding): UntilLoop { val untilCondition = expression().toAst(encoding) val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) val scope = AnonymousScope(statements, statement_block()?.toPosition() @@ -580,13 +583,13 @@ private fun prog8Parser.UntilloopContext.toAst(encoding: IStringEncoding): Until return UntilLoop(scope, untilCondition, toPosition()) } -private fun prog8Parser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { +private fun Prog8ANTLRParser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { val condition = expression().toAst(encoding) val choices = this.when_choice()?.map { it.toAst(encoding) }?.toMutableList() ?: mutableListOf() return WhenStatement(condition, choices, toPosition()) } -private fun prog8Parser.When_choiceContext.toAst(encoding: IStringEncoding): WhenChoice { +private fun Prog8ANTLRParser.When_choiceContext.toAst(encoding: IStringEncoding): WhenChoice { val values = expression_list()?.toAst(encoding) val stmt = statement()?.toAst(encoding) val stmtBlock = statement_block()?.toAst(encoding)?.toMutableList() ?: mutableListOf() @@ -596,7 +599,7 @@ private fun prog8Parser.When_choiceContext.toAst(encoding: IStringEncoding): Whe return WhenChoice(values?.toMutableList(), scope, toPosition()) } -private fun prog8Parser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl { +private fun Prog8ANTLRParser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl { return VarDecl( VarDeclType.VAR, datatype()?.toAst() ?: DataType.UNDEFINED, diff --git a/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt b/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt index efb769bf3..dd73a29cc 100644 --- a/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt +++ b/compilerAst/src/prog8/parser/CommentHandlingTokenStream.kt @@ -9,7 +9,7 @@ internal class CommentHandlingTokenStream(lexer: Lexer) : CommonTokenStream(lexe fun commentTokens() : List { // extract the comments - val commentTokenChannel = prog8Lexer.channelNames.indexOf("HIDDEN") + val commentTokenChannel = Prog8ANTLRLexer.channelNames.indexOf("HIDDEN") val theLexer = tokenSource as Lexer return get(0, size()) .asSequence() diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 94faa6ee4..d9a220344 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -18,8 +18,6 @@ import java.nio.file.Paths class ParsingFailedError(override var message: String) : Exception(message) -internal class CustomLexer(val modulePath: Path, input: CharStream?) : prog8Lexer(input) - fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') internal fun pathFrom(stringPath: String, vararg rest: String): Path = FileSystems.getDefault().getPath(stringPath, *rest) @@ -44,7 +42,10 @@ class ModuleImporter(private val program: Program, if(!Files.isReadable(filePath)) throw ParsingFailedError("No such file: $filePath") - return importModule(CharStreams.fromPath(filePath), filePath) + val content = filePath.toFile().readText() + val cs = CharStreams.fromString(content) + + return importModule(cs, filePath) } fun importLibraryModule(name: String): Module? { @@ -54,40 +55,10 @@ class ModuleImporter(private val program: Program, return executeImportDirective(import, Paths.get("")) } - private class MyErrorListener: ConsoleErrorListener() { - var numberOfErrors: Int = 0 - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - numberOfErrors++ - when (recognizer) { - is CustomLexer -> System.err.println("${recognizer.modulePath}:$line:$charPositionInLine: $msg") - is prog8Parser -> System.err.println("${recognizer.inputStream.sourceName}:$line:$charPositionInLine: $msg") - else -> System.err.println("$line:$charPositionInLine $msg") - } - if(numberOfErrors>=5) - throw ParsingFailedError("There are too many parse errors. Stopping.") - } - } - private fun importModule(stream: CharStream, modulePath: Path): Module { - val moduleName = moduleName(modulePath.fileName) - val lexer = CustomLexer(modulePath, stream) - lexer.removeErrorListeners() - val lexerErrors = MyErrorListener() - lexer.addErrorListener(lexerErrors) - val tokens = CommentHandlingTokenStream(lexer) - val parser = prog8Parser(tokens) - parser.removeErrorListeners() - parser.addErrorListener(MyErrorListener()) - val parseTree = parser.module() - val numberOfErrors = parser.numberOfSyntaxErrors + lexerErrors.numberOfErrors - if(numberOfErrors > 0) - throw ParsingFailedError("There are $numberOfErrors errors in '$moduleName'.") - - // You can do something with the parsed comments: - // tokens.commentTokens().forEach { println(it) } - - // convert to Ast - val moduleAst = parseTree.toAst(moduleName, modulePath, encoder) + val parser = Prog8Parser() + val sourceText = stream.toString() + val moduleAst = parser.parseModule(sourceText) moduleAst.program = program moduleAst.linkParents(program.namespace) program.modules.add(moduleAst) diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt new file mode 100644 index 000000000..51b60ef44 --- /dev/null +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -0,0 +1,65 @@ +package prog8.parser + +import org.antlr.v4.runtime.* +import org.antlr.v4.runtime.misc.ParseCancellationException +import prog8.ast.antlr.toAst +import prog8.ast.Module +import prog8.ast.base.Position +import prog8.ast.IStringEncoding + + +object DummyEncoding: IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("move StringEncoding out of compilerAst") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("move StringEncoding out of compilerAst") + } +} + +class Prog8ErrorStrategy: BailErrorStrategy() { + override fun recover(recognizer: Parser?, e: RecognitionException?) { + try { + // let it + super.recover(recognizer, e) // fills in exception e in all the contexts + // ...then throws ParseCancellationException, which is + // *deliberately* not a RecognitionException. However, we don't try any + // error recovery, therefore report an error in this case, too. + } catch (pce: ParseCancellationException) { + reportError(recognizer, e) + } + } + + override fun recoverInline(recognizer: Parser?): Token { + throw InputMismatchException(recognizer) + } +} + +object ThrowErrorListener: BaseErrorListener() { + override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { + throw ParsingFailedError("$e: $msg") + } +} + +class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorListener) { + + fun parseModule(sourceText: String): Module { + val chars = CharStreams.fromString(sourceText) + val lexer = Prog8ANTLRLexer(chars) + lexer.removeErrorListeners() + lexer.addErrorListener(errorListener) + val tokens = CommonTokenStream(lexer) + val parser = Prog8ANTLRParser(tokens) + parser.errorHandler = Prog8ErrorStrategy() + parser.removeErrorListeners() + parser.addErrorListener(errorListener) + + val parseTree = parser.module() + + // TODO: use Module ctor directly + val moduleAst = parseTree.toAst("anonymous", pathFrom(""), DummyEncoding) + + return moduleAst + } +} diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestAntlrParser.kt index bb90bfb34..9034685a4 100644 --- a/compilerAst/test/TestAntlrParser.kt +++ b/compilerAst/test/TestAntlrParser.kt @@ -29,8 +29,11 @@ class TestAntlrParser { class MyErrorStrategy: BailErrorStrategy() { override fun recover(recognizer: Parser?, e: RecognitionException?) { try { - // let it fill in e in all the contexts - super.recover(recognizer, e) + // let it + super.recover(recognizer, e) // fills in exception e in all the contexts + // ...then throws ParseCancellationException, which is + // *deliberately* not a RecognitionException. However, we don't try any + // error recovery, therefore report an error in this case, too. } catch (pce: ParseCancellationException) { reportError(recognizer, e) } @@ -41,21 +44,21 @@ class TestAntlrParser { } } - private fun parseModule(srcText: String): prog8Parser.ModuleContext { + private fun parseModule(srcText: String): Prog8ANTLRParser.ModuleContext { return parseModule(CharStreams.fromString(srcText)) } - private fun parseModule(srcFile: Path): prog8Parser.ModuleContext { + private fun parseModule(srcFile: Path): Prog8ANTLRParser.ModuleContext { return parseModule(CharStreams.fromPath(srcFile)) } - private fun parseModule(srcStream: CharStream): prog8Parser.ModuleContext { + private fun parseModule(srcStream: CharStream): Prog8ANTLRParser.ModuleContext { val errorListener = MyErrorListener() - val lexer = prog8Lexer(srcStream) + val lexer = Prog8ANTLRLexer(srcStream) lexer.removeErrorListeners() lexer.addErrorListener(errorListener) val tokens = CommonTokenStream(lexer) - val parser = prog8Parser(tokens) + val parser = Prog8ANTLRParser(tokens) parser.errorHandler = MyErrorStrategy() parser.removeErrorListeners() parser.addErrorListener(errorListener) @@ -88,7 +91,7 @@ class TestAntlrParser { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) - // before the fix, prog8Parser would have reported (thrown) "missing at ''" + // before the fix, Prog8ANTLRParser would have reported (thrown) "missing at ''" val parseTree = parseModule(srcText) assertEquals(parseTree.block().size, 1) } @@ -234,22 +237,14 @@ class TestAntlrParser { @Test fun testProg8Ast() { - // can create charstreams from many other sources as well; - val charstream = CharStreams.fromString(""" + val parseTree = parseModule(""" main { sub start() { return } } """) - val lexer = prog8Lexer(charstream) - val tokens = CommonTokenStream(lexer) - val parser = prog8Parser(tokens) - parser.errorHandler = BailErrorStrategy() -// parser.removeErrorListeners() -// parser.addErrorListener(MyErrorListener()) - - val ast = parser.module().toAst("test", Path.of(""), DummyEncoding) + val ast = parseTree.toAst("test", Path.of(""), DummyEncoding) assertIs(ast.statements.first()) assertEquals((ast.statements.first() as Block).name, "main") } diff --git a/parser/antlr/prog8.g4 b/parser/antlr/Prog8ANTLR.g4 similarity index 97% rename from parser/antlr/prog8.g4 rename to parser/antlr/Prog8ANTLR.g4 index 9a2cc4fe4..af05c0aec 100644 --- a/parser/antlr/prog8.g4 +++ b/parser/antlr/Prog8ANTLR.g4 @@ -8,7 +8,9 @@ NOTES: */ -grammar prog8; +// -> java classes Prog8ANTLRParser and Prog8ANTLRLexer, +// both NOT to be used from Kotlin code, but ONLY through Kotlin class Prog8Parser +grammar Prog8ANTLR; @header { package prog8.parser; From 46911a890574dac6471f4946e8d6f1c1c95a5c73 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 3 Jul 2021 16:49:46 +0200 Subject: [PATCH 02/68] + temporarily add PetsciiEncoding (and Petscii.kt copied from compiler) to parser; .linkParents for child nodes of Module --- compilerAst/src/prog8/parser/Petscii.kt | 1171 +++++++++++++++++ .../src/prog8/parser/PetsciiEncoding.kt | 24 + compilerAst/src/prog8/parser/Prog8Parser.kt | 20 +- .../src/prog8/parser/ThrowTodoEncoding.kt | 16 + 4 files changed, 1217 insertions(+), 14 deletions(-) create mode 100644 compilerAst/src/prog8/parser/Petscii.kt create mode 100644 compilerAst/src/prog8/parser/PetsciiEncoding.kt create mode 100644 compilerAst/src/prog8/parser/ThrowTodoEncoding.kt diff --git a/compilerAst/src/prog8/parser/Petscii.kt b/compilerAst/src/prog8/parser/Petscii.kt new file mode 100644 index 000000000..c79277af3 --- /dev/null +++ b/compilerAst/src/prog8/parser/Petscii.kt @@ -0,0 +1,1171 @@ +package prog8.parser + +import prog8.ast.antlr.escape +import java.io.CharConversionException + +object Petscii { + + // decoding: from Petscii/Screencodes (0-255) to unicode + // character tables used from https://github.com/dj51d/cbmcodecs + + private val decodingPetsciiLowercase = arrayOf( + '\u0000', // 0x00 -> \u0000 + '\ufffe', // 0x01 -> UNDEFINED + '\ufffe', // 0x02 -> UNDEFINED + '\ufffe', // 0x03 -> UNDEFINED + '\ufffe', // 0x04 -> UNDEFINED + '\uf100', // 0x05 -> WHITE COLOR SWITCH (CUS) + '\ufffe', // 0x06 -> UNDEFINED + '\ufffe', // 0x07 -> UNDEFINED + '\uf118', // 0x08 -> DISABLE CHARACTER SET SWITCHING (CUS) + '\uf119', // 0x09 -> ENABLE CHARACTER SET SWITCHING (CUS) + '\ufffe', // 0x0A -> UNDEFINED + '\ufffe', // 0x0B -> UNDEFINED + '\ufffe', // 0x0C -> UNDEFINED + '\r' , // 0x0D -> CARRIAGE RETURN + '\u000e', // 0x0E -> SHIFT OUT + '\ufffe', // 0x0F -> UNDEFINED + '\ufffe', // 0x10 -> UNDEFINED + '\uf11c', // 0x11 -> CURSOR DOWN (CUS) + '\uf11a', // 0x12 -> REVERSE VIDEO ON (CUS) + '\uf120', // 0x13 -> HOME (CUS) + '\u007f', // 0x14 -> DELETE + '\ufffe', // 0x15 -> UNDEFINED + '\ufffe', // 0x16 -> UNDEFINED + '\ufffe', // 0x17 -> UNDEFINED + '\ufffe', // 0x18 -> UNDEFINED + '\ufffe', // 0x19 -> UNDEFINED + '\ufffe', // 0x1A -> UNDEFINED + '\ufffe', // 0x1B -> UNDEFINED + '\uf101', // 0x1C -> RED COLOR SWITCH (CUS) + '\uf11d', // 0x1D -> CURSOR RIGHT (CUS) + '\uf102', // 0x1E -> GREEN COLOR SWITCH (CUS) + '\uf103', // 0x1F -> BLUE COLOR SWITCH (CUS) + ' ' , // 0x20 -> SPACE + '!' , // ! 0x21 -> EXCLAMATION MARK + '"' , // " 0x22 -> QUOTATION MARK + '#' , // # 0x23 -> NUMBER SIGN + '$' , // $ 0x24 -> DOLLAR SIGN + '%' , // % 0x25 -> PERCENT SIGN + '&' , // & 0x26 -> AMPERSAND + '\'' , // ' 0x27 -> APOSTROPHE + '(' , // ( 0x28 -> LEFT PARENTHESIS + ')' , // ) 0x29 -> RIGHT PARENTHESIS + '*' , // * 0x2A -> ASTERISK + '+' , // + 0x2B -> PLUS SIGN + ',' , // , 0x2C -> COMMA + '-' , // - 0x2D -> HYPHEN-MINUS + '.' , // . 0x2E -> FULL STOP + '/' , // / 0x2F -> SOLIDUS + '0' , // 0 0x30 -> DIGIT ZERO + '1' , // 1 0x31 -> DIGIT ONE + '2' , // 2 0x32 -> DIGIT TWO + '3' , // 3 0x33 -> DIGIT THREE + '4' , // 4 0x34 -> DIGIT FOUR + '5' , // 5 0x35 -> DIGIT FIVE + '6' , // 6 0x36 -> DIGIT SIX + '7' , // 7 0x37 -> DIGIT SEVEN + '8' , // 8 0x38 -> DIGIT EIGHT + '9' , // 9 0x39 -> DIGIT NINE + ':' , // : 0x3A -> COLON + ';' , // ; 0x3B -> SEMICOLON + '<' , // < 0x3C -> LESS-THAN SIGN + '=' , // = 0x3D -> EQUALS SIGN + '>' , // > 0x3E -> GREATER-THAN SIGN + '?' , // ? 0x3F -> QUESTION MARK + '@' , // @ 0x40 -> COMMERCIAL AT + 'a' , // a 0x41 -> LATIN SMALL LETTER A + 'b' , // b 0x42 -> LATIN SMALL LETTER B + 'c' , // c 0x43 -> LATIN SMALL LETTER C + 'd' , // d 0x44 -> LATIN SMALL LETTER D + 'e' , // e 0x45 -> LATIN SMALL LETTER E + 'f' , // f 0x46 -> LATIN SMALL LETTER F + 'g' , // g 0x47 -> LATIN SMALL LETTER G + 'h' , // h 0x48 -> LATIN SMALL LETTER H + 'i' , // i 0x49 -> LATIN SMALL LETTER I + 'j' , // j 0x4A -> LATIN SMALL LETTER J + 'k' , // k 0x4B -> LATIN SMALL LETTER K + 'l' , // l 0x4C -> LATIN SMALL LETTER L + 'm' , // m 0x4D -> LATIN SMALL LETTER M + 'n' , // n 0x4E -> LATIN SMALL LETTER N + 'o' , // o 0x4F -> LATIN SMALL LETTER O + 'p' , // p 0x50 -> LATIN SMALL LETTER P + 'q' , // q 0x51 -> LATIN SMALL LETTER Q + 'r' , // r 0x52 -> LATIN SMALL LETTER R + 's' , // s 0x53 -> LATIN SMALL LETTER S + 't' , // t 0x54 -> LATIN SMALL LETTER T + 'u' , // u 0x55 -> LATIN SMALL LETTER U + 'v' , // v 0x56 -> LATIN SMALL LETTER V + 'w' , // w 0x57 -> LATIN SMALL LETTER W + 'x' , // x 0x58 -> LATIN SMALL LETTER X + 'y' , // y 0x59 -> LATIN SMALL LETTER Y + 'z' , // z 0x5A -> LATIN SMALL LETTER Z + '[' , // [ 0x5B -> LEFT SQUARE BRACKET + '\u00a3', // £ 0x5C -> POUND SIGN + ']' , // ] 0x5D -> RIGHT SQUARE BRACKET + '\u2191', // ↑ 0x5E -> UPWARDS ARROW + '\u2190', // ← 0x5F -> LEFTWARDS ARROW + '\u2500', // ─ 0x60 -> BOX DRAWINGS LIGHT HORIZONTAL + 'A' , // A 0x61 -> LATIN CAPITAL LETTER A + 'B' , // B 0x62 -> LATIN CAPITAL LETTER B + 'C' , // C 0x63 -> LATIN CAPITAL LETTER C + 'D' , // D 0x64 -> LATIN CAPITAL LETTER D + 'E' , // E 0x65 -> LATIN CAPITAL LETTER E + 'F' , // F 0x66 -> LATIN CAPITAL LETTER F + 'G' , // G 0x67 -> LATIN CAPITAL LETTER G + 'H' , // H 0x68 -> LATIN CAPITAL LETTER H + 'I' , // I 0x69 -> LATIN CAPITAL LETTER I + 'J' , // J 0x6A -> LATIN CAPITAL LETTER J + 'K' , // K 0x6B -> LATIN CAPITAL LETTER K + 'L' , // L 0x6C -> LATIN CAPITAL LETTER L + 'M' , // M 0x6D -> LATIN CAPITAL LETTER M + 'N' , // N 0x6E -> LATIN CAPITAL LETTER N + 'O' , // O 0x6F -> LATIN CAPITAL LETTER O + 'P' , // P 0x70 -> LATIN CAPITAL LETTER P + 'Q' , // Q 0x71 -> LATIN CAPITAL LETTER Q + 'R' , // R 0x72 -> LATIN CAPITAL LETTER R + 'S' , // S 0x73 -> LATIN CAPITAL LETTER S + 'T' , // T 0x74 -> LATIN CAPITAL LETTER T + 'U' , // U 0x75 -> LATIN CAPITAL LETTER U + 'V' , // V 0x76 -> LATIN CAPITAL LETTER V + 'W' , // W 0x77 -> LATIN CAPITAL LETTER W + 'X' , // X 0x78 -> LATIN CAPITAL LETTER X + 'Y' , // Y 0x79 -> LATIN CAPITAL LETTER Y + 'Z' , // Z 0x7A -> LATIN CAPITAL LETTER Z + '\u253c', // ┼ 0x7B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0x7C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0x7D -> BOX DRAWINGS LIGHT VERTICAL + '\u2592', // ▒ 0x7E -> MEDIUM SHADE + '\uf139', //  0x7F -> MEDIUM SHADE SLASHED LEFT (CUS) + '\ufffe', // 0x80 -> UNDEFINED + '\uf104', // 0x81 -> ORANGE COLOR SWITCH (CUS) + '\ufffe', // 0x82 -> UNDEFINED + '\ufffe', // 0x83 -> UNDEFINED + '\ufffe', // 0x84 -> UNDEFINED + '\uf110', //  0x85 -> FUNCTION KEY 1 (CUS) + '\uf112', //  0x86 -> FUNCTION KEY 3 (CUS) + '\uf114', //  0x87 -> FUNCTION KEY 5 (CUS) + '\uf116', //  0x88 -> FUNCTION KEY 7 (CUS) + '\uf111', //  0x89 -> FUNCTION KEY 2 (CUS) + '\uf113', //  0x8A -> FUNCTION KEY 4 (CUS) + '\uf115', //  0x8B -> FUNCTION KEY 6 (CUS) + '\uf117', //  0x8C -> FUNCTION KEY 8 (CUS) + '\n' , // 0x8D -> LINE FEED + '\u000f', //  0x8E -> SHIFT IN + '\ufffe', // 0x8F -> UNDEFINED + '\uf105', // 0x90 -> BLACK COLOR SWITCH (CUS) + '\uf11e', //  0x91 -> CURSOR UP (CUS) + '\uf11b', //  0x92 -> REVERSE VIDEO OFF (CUS) + '\u000c', // 0x93 -> FORM FEED + '\uf121', //  0x94 -> INSERT (CUS) + '\uf106', // 0x95 -> BROWN COLOR SWITCH (CUS) + '\uf107', // 0x96 -> LIGHT RED COLOR SWITCH (CUS) + '\uf108', // 0x97 -> GRAY 1 COLOR SWITCH (CUS) + '\uf109', //  0x98 -> GRAY 2 COLOR SWITCH (CUS) + '\uf10a', //  0x99 -> LIGHT GREEN COLOR SWITCH (CUS) + '\uf10b', //  0x9A -> LIGHT BLUE COLOR SWITCH (CUS) + '\uf10c', //  0x9B -> GRAY 3 COLOR SWITCH (CUS) + '\uf10d', //  0x9C -> PURPLE COLOR SWITCH (CUS) + '\uf11d', //  0x9D -> CURSOR LEFT (CUS) + '\uf10e', //  0x9E -> YELLOW COLOR SWITCH (CUS) + '\uf10f', //  0x9F -> CYAN COLOR SWITCH (CUS) + '\u00a0', // 0xA0 -> NO-BREAK SPACE + '\u258c', // ▌ 0xA1 -> LEFT HALF BLOCK + '\u2584', // ▄ 0xA2 -> LOWER HALF BLOCK + '\u2594', // ▔ 0xA3 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0xA4 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0xA5 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0xA6 -> MEDIUM SHADE + '\u2595', // ▕ 0xA7 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0xA8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\uf13a', //  0xA9 -> MEDIUM SHADE SLASHED RIGHT (CUS) + '\uf130', //  0xAA -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0xAB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0xAC -> QUADRANT LOWER RIGHT + '\u2514', // └ 0xAD -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0xAE -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0xAF -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0xB0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0xB1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0xB2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0xB3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0xB4 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0xB5 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0xB6 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0xB7 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0xB8 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0xB9 -> LOWER THREE EIGHTHS BLOCK + '\u2713', // ✓ 0xBA -> CHECK MARK + '\u2596', // ▖ 0xBB -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0xBC -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0xBD -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0xBE -> QUADRANT UPPER LEFT + '\u259a', // ▚ 0xBF -> QUADRANT UPPER LEFT AND LOWER RIGHT + '\u2500', // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL + 'A' , // A 0xC1 -> LATIN CAPITAL LETTER A + 'B' , // B 0xC2 -> LATIN CAPITAL LETTER B + 'C' , // C 0xC3 -> LATIN CAPITAL LETTER C + 'D' , // D 0xC4 -> LATIN CAPITAL LETTER D + 'E' , // E 0xC5 -> LATIN CAPITAL LETTER E + 'F' , // F 0xC6 -> LATIN CAPITAL LETTER F + 'G' , // G 0xC7 -> LATIN CAPITAL LETTER G + 'H' , // H 0xC8 -> LATIN CAPITAL LETTER H + 'I' , // I 0xC9 -> LATIN CAPITAL LETTER I + 'J' , // J 0xCA -> LATIN CAPITAL LETTER J + 'K' , // K 0xCB -> LATIN CAPITAL LETTER K + 'L' , // L 0xCC -> LATIN CAPITAL LETTER L + 'M' , // M 0xCD -> LATIN CAPITAL LETTER M + 'N' , // N 0xCE -> LATIN CAPITAL LETTER N + 'O' , // O 0xCF -> LATIN CAPITAL LETTER O + 'P' , // P 0xD0 -> LATIN CAPITAL LETTER P + 'Q' , // Q 0xD1 -> LATIN CAPITAL LETTER Q + 'R' , // R 0xD2 -> LATIN CAPITAL LETTER R + 'S' , // S 0xD3 -> LATIN CAPITAL LETTER S + 'T' , // T 0xD4 -> LATIN CAPITAL LETTER T + 'U' , // U 0xD5 -> LATIN CAPITAL LETTER U + 'V' , // V 0xD6 -> LATIN CAPITAL LETTER V + 'W' , // W 0xD7 -> LATIN CAPITAL LETTER W + 'X' , // X 0xD8 -> LATIN CAPITAL LETTER X + 'Y' , // Y 0xD9 -> LATIN CAPITAL LETTER Y + 'Z' , // Z 0xDA -> LATIN CAPITAL LETTER Z + '\u253c', // ┼ 0xDB -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0xDC -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0xDD -> BOX DRAWINGS LIGHT VERTICAL + '\u2592', // ▒ 0xDE -> MEDIUM SHADE + '\uf139', //  0xDF -> MEDIUM SHADE SLASHED LEFT (CUS) + '\u00a0', // 0xE0 -> NO-BREAK SPACE + '\u258c', // ▌ 0xE1 -> LEFT HALF BLOCK + '\u2584', // ▄ 0xE2 -> LOWER HALF BLOCK + '\u2594', // ▔ 0xE3 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0xE4 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0xE5 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0xE6 -> MEDIUM SHADE + '\u2595', // ▕ 0xE7 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0xE8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\uf13a', //  0xE9 -> MEDIUM SHADE SLASHED RIGHT (CUS) + '\uf130', //  0xEA -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0xEB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0xEC -> QUADRANT LOWER RIGHT + '\u2514', // └ 0xED -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0xEE -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0xEF -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0xF0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0xF1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0xF2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0xF3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0xF4 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0xF5 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0xF6 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0xF7 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0xF8 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0xF9 -> LOWER THREE EIGHTHS BLOCK + '\u2713', // ✓ 0xFA -> CHECK MARK + '\u2596', // ▖ 0xFB -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0xFC -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0xFD -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0xFE -> QUADRANT UPPER LEFT + '\u2592' // ▒ 0xFF -> MEDIUM SHADE + ) + + private val decodingPetsciiUppercase = arrayOf( + '\u0000', // 0x00 -> \u0000 + '\ufffe', // 0x01 -> UNDEFINED + '\ufffe', // 0x02 -> UNDEFINED + '\ufffe', // 0x03 -> UNDEFINED + '\ufffe', // 0x04 -> UNDEFINED + '\uf100', // 0x05 -> WHITE COLOR SWITCH (CUS) + '\ufffe', // 0x06 -> UNDEFINED + '\ufffe', // 0x07 -> UNDEFINED + '\uf118', // 0x08 -> DISABLE CHARACTER SET SWITCHING (CUS) + '\uf119', // 0x09 -> ENABLE CHARACTER SET SWITCHING (CUS) + '\ufffe', // 0x0A -> UNDEFINED + '\ufffe', // 0x0B -> UNDEFINED + '\ufffe', // 0x0C -> UNDEFINED + '\r' , // 0x0D -> CARRIAGE RETURN + '\u000e', // 0x0E -> SHIFT OUT + '\ufffe', // 0x0F -> UNDEFINED + '\ufffe', // 0x10 -> UNDEFINED + '\uf11c', // 0x11 -> CURSOR DOWN (CUS) + '\uf11a', // 0x12 -> REVERSE VIDEO ON (CUS) + '\uf120', // 0x13 -> HOME (CUS) + '\u007f', // 0x14 -> DELETE + '\ufffe', // 0x15 -> UNDEFINED + '\ufffe', // 0x16 -> UNDEFINED + '\ufffe', // 0x17 -> UNDEFINED + '\ufffe', // 0x18 -> UNDEFINED + '\ufffe', // 0x19 -> UNDEFINED + '\ufffe', // 0x1A -> UNDEFINED + '\ufffe', // 0x1B -> UNDEFINED + '\uf101', // 0x1C -> RED COLOR SWITCH (CUS) + '\uf11d', // 0x1D -> CURSOR RIGHT (CUS) + '\uf102', // 0x1E -> GREEN COLOR SWITCH (CUS) + '\uf103', // 0x1F -> BLUE COLOR SWITCH (CUS) + ' ' , // 0x20 -> SPACE + '!' , // ! 0x21 -> EXCLAMATION MARK + '"' , // " 0x22 -> QUOTATION MARK + '#' , // # 0x23 -> NUMBER SIGN + '$' , // $ 0x24 -> DOLLAR SIGN + '%' , // % 0x25 -> PERCENT SIGN + '&' , // & 0x26 -> AMPERSAND + '\'' , // ' 0x27 -> APOSTROPHE + '(' , // ( 0x28 -> LEFT PARENTHESIS + ')' , // ) 0x29 -> RIGHT PARENTHESIS + '*' , // * 0x2A -> ASTERISK + '+' , // + 0x2B -> PLUS SIGN + ',' , // , 0x2C -> COMMA + '-' , // - 0x2D -> HYPHEN-MINUS + '.' , // . 0x2E -> FULL STOP + '/' , // / 0x2F -> SOLIDUS + '0' , // 0 0x30 -> DIGIT ZERO + '1' , // 1 0x31 -> DIGIT ONE + '2' , // 2 0x32 -> DIGIT TWO + '3' , // 3 0x33 -> DIGIT THREE + '4' , // 4 0x34 -> DIGIT FOUR + '5' , // 5 0x35 -> DIGIT FIVE + '6' , // 6 0x36 -> DIGIT SIX + '7' , // 7 0x37 -> DIGIT SEVEN + '8' , // 8 0x38 -> DIGIT EIGHT + '9' , // 9 0x39 -> DIGIT NINE + ':' , // : 0x3A -> COLON + ';' , // ; 0x3B -> SEMICOLON + '<' , // < 0x3C -> LESS-THAN SIGN + '=' , // = 0x3D -> EQUALS SIGN + '>' , // > 0x3E -> GREATER-THAN SIGN + '?' , // ? 0x3F -> QUESTION MARK + '@' , // @ 0x40 -> COMMERCIAL AT + 'A' , // A 0x41 -> LATIN CAPITAL LETTER A + 'B' , // B 0x42 -> LATIN CAPITAL LETTER B + 'C' , // C 0x43 -> LATIN CAPITAL LETTER C + 'D' , // D 0x44 -> LATIN CAPITAL LETTER D + 'E' , // E 0x45 -> LATIN CAPITAL LETTER E + 'F' , // F 0x46 -> LATIN CAPITAL LETTER F + 'G' , // G 0x47 -> LATIN CAPITAL LETTER G + 'H' , // H 0x48 -> LATIN CAPITAL LETTER H + 'I' , // I 0x49 -> LATIN CAPITAL LETTER I + 'J' , // J 0x4A -> LATIN CAPITAL LETTER J + 'K' , // K 0x4B -> LATIN CAPITAL LETTER K + 'L' , // L 0x4C -> LATIN CAPITAL LETTER L + 'M' , // M 0x4D -> LATIN CAPITAL LETTER M + 'N' , // N 0x4E -> LATIN CAPITAL LETTER N + 'O' , // O 0x4F -> LATIN CAPITAL LETTER O + 'P' , // P 0x50 -> LATIN CAPITAL LETTER P + 'Q' , // Q 0x51 -> LATIN CAPITAL LETTER Q + 'R' , // R 0x52 -> LATIN CAPITAL LETTER R + 'S' , // S 0x53 -> LATIN CAPITAL LETTER S + 'T' , // T 0x54 -> LATIN CAPITAL LETTER T + 'U' , // U 0x55 -> LATIN CAPITAL LETTER U + 'V' , // V 0x56 -> LATIN CAPITAL LETTER V + 'W' , // W 0x57 -> LATIN CAPITAL LETTER W + 'X' , // X 0x58 -> LATIN CAPITAL LETTER X + 'Y' , // Y 0x59 -> LATIN CAPITAL LETTER Y + 'Z' , // Z 0x5A -> LATIN CAPITAL LETTER Z + '[' , // [ 0x5B -> LEFT SQUARE BRACKET + '\u00a3', // £ 0x5C -> POUND SIGN + ']' , // ] 0x5D -> RIGHT SQUARE BRACKET + '\u2191', // ↑ 0x5E -> UPWARDS ARROW + '\u2190', // ← 0x5F -> LEFTWARDS ARROW + '\u2500', // ─ 0x60 -> BOX DRAWINGS LIGHT HORIZONTAL + '\u2660', // ♠ 0x61 -> BLACK SPADE SUIT + '\u2502', // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL + '\u2500', // ─ 0x63 -> BOX DRAWINGS LIGHT HORIZONTAL + '\uf122', //  0x64 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) + '\uf123', //  0x65 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) + '\uf124', //  0x66 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) + '\uf126', //  0x67 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) + '\uf128', //  0x68 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) + '\u256e', // ╮ 0x69 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT + '\u2570', // ╰ 0x6A -> BOX DRAWINGS LIGHT ARC UP AND RIGHT + '\u256f', // ╯ 0x6B -> BOX DRAWINGS LIGHT ARC UP AND LEFT + '\uf12a', //  0x6C -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) + '\u2572', // ╲ 0x6D -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + '\u2571', // ╱ 0x6E -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + '\uf12b', //  0x6F -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) + '\uf12c', //  0x70 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) + '\u25cf', // ● 0x71 -> BLACK CIRCLE + '\uf125', //  0x72 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) + '\u2665', // ♥ 0x73 -> BLACK HEART SUIT + '\uf127', //  0x74 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) + '\u256d', // ╭ 0x75 -> BOX DRAWINGS LIGHT ARC DOWN AND RIGHT + '\u2573', // ╳ 0x76 -> BOX DRAWINGS LIGHT DIAGONAL CROSS + '\u25cb', // ○ 0x77 -> WHITE CIRCLE + '\u2663', // ♣ 0x78 -> BLACK CLUB SUIT + '\uf129', //  0x79 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) + '\u2666', // ♦ 0x7A -> BLACK DIAMOND SUIT + '\u253c', // ┼ 0x7B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0x7C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0x7D -> BOX DRAWINGS LIGHT VERTICAL + '\u03c0', // π 0x7E -> GREEK SMALL LETTER PI + '\u25e5', // ◥ 0x7F -> BLACK UPPER RIGHT TRIANGLE + '\ufffe', // 0x80 -> UNDEFINED + '\uf104', //  0x81 -> ORANGE COLOR SWITCH (CUS) + '\ufffe', // 0x82 -> UNDEFINED + '\ufffe', // 0x83 -> UNDEFINED + '\ufffe', // 0x84 -> UNDEFINED + '\uf110', // 0x85 -> FUNCTION KEY 1 (CUS) + '\uf112', // 0x86 -> FUNCTION KEY 3 (CUS) + '\uf114', // 0x87 -> FUNCTION KEY 5 (CUS) + '\uf116', // 0x88 -> FUNCTION KEY 7 (CUS) + '\uf111', // 0x89 -> FUNCTION KEY 2 (CUS) + '\uf113', // 0x8A -> FUNCTION KEY 4 (CUS) + '\uf115', // 0x8B -> FUNCTION KEY 6 (CUS) + '\uf117', // 0x8C -> FUNCTION KEY 8 (CUS) + '\n' , // 0x8D -> LINE FEED + '\u000f', // 0x8E -> SHIFT IN + '\ufffe', // 0x8F -> UNDEFINED + '\uf105', // 0x90 -> BLACK COLOR SWITCH (CUS) + '\uf11e', // 0x91 -> CURSOR UP (CUS) + '\uf11b', // 0x92 -> REVERSE VIDEO OFF (CUS) + '\u000c', // 0x93 -> FORM FEED + '\uf121', // 0x94 -> INSERT (CUS) + '\uf106', // 0x95 -> BROWN COLOR SWITCH (CUS) + '\uf107', // 0x96 -> LIGHT RED COLOR SWITCH (CUS) + '\uf108', // 0x97 -> GRAY 1 COLOR SWITCH (CUS) + '\uf109', // 0x98 -> GRAY 2 COLOR SWITCH (CUS) + '\uf10a', // 0x99 -> LIGHT GREEN COLOR SWITCH (CUS) + '\uf10b', // 0x9A -> LIGHT BLUE COLOR SWITCH (CUS) + '\uf10c', // 0x9B -> GRAY 3 COLOR SWITCH (CUS) + '\uf10d', // 0x9C -> PURPLE COLOR SWITCH (CUS) + '\uf11d', // 0x9D -> CURSOR LEFT (CUS) + '\uf10e', // 0x9E -> YELLOW COLOR SWITCH (CUS) + '\uf10f', // 0x9F -> CYAN COLOR SWITCH (CUS) + '\u00a0', // 0xA0 -> NO-BREAK SPACE + '\u258c', // ▌ 0xA1 -> LEFT HALF BLOCK + '\u2584', // ▄ 0xA2 -> LOWER HALF BLOCK + '\u2594', // ▔ 0xA3 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0xA4 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0xA5 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0xA6 -> MEDIUM SHADE + '\u2595', // ▕ 0xA7 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0xA8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\u25e4', // ◤ 0xA9 -> BLACK UPPER LEFT TRIANGLE + '\uf130', //  0xAA -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0xAB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0xAC -> QUADRANT LOWER RIGHT + '\u2514', // └ 0xAD -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0xAE -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0xAF -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0xB0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0xB1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0xB2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0xB3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0xB4 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0xB5 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0xB6 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0xB7 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0xB8 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0xB9 -> LOWER THREE EIGHTHS BLOCK + '\uf12d', //  0xBA -> ONE EIGHTH BLOCK UP AND LEFT (CUS) + '\u2596', // ▖ 0xBB -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0xBC -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0xBD -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0xBE -> QUADRANT UPPER LEFT + '\u259a', // ▚ 0xBF -> QUADRANT UPPER LEFT AND LOWER RIGHT + '\u2500', // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL + '\u2660', // ♠ 0xC1 -> BLACK SPADE SUIT + '\u2502', // │ 0xC2 -> BOX DRAWINGS LIGHT VERTICAL + '\u2500', // ─ 0xC3 -> BOX DRAWINGS LIGHT HORIZONTAL + '\uf122', //  0xC4 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) + '\uf123', //  0xC5 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) + '\uf124', //  0xC6 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) + '\uf126', //  0xC7 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) + '\uf128', //  0xC8 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) + '\u256e', // ╮ 0xC9 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT + '\u2570', // ╰ 0xCA -> BOX DRAWINGS LIGHT ARC UP AND RIGHT + '\u256f', // ╯ 0xCB -> BOX DRAWINGS LIGHT ARC UP AND LEFT + '\uf12a', //  0xCC -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) + '\u2572', // ╲ 0xCD -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + '\u2571', // ╱ 0xCE -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + '\uf12b', //  0xCF -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) + '\uf12c', //  0xD0 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) + '\u25cf', // ● 0xD1 -> BLACK CIRCLE + '\uf125', //  0xD2 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) + '\u2665', // ♥ 0xD3 -> BLACK HEART SUIT + '\uf127', //  0xD4 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) + '\u256d', // ╭ 0xD5 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT + '\u2573', // ╳ 0xD6 -> BOX DRAWINGS LIGHT DIAGONAL CROSS + '\u25cb', // ○ 0xD7 -> WHITE CIRCLE + '\u2663', // ♣ 0xD8 -> BLACK CLUB SUIT + '\uf129', //  0xD9 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) + '\u2666', // ♦ 0xDA -> BLACK DIAMOND SUIT + '\u253c', // ┼ 0xDB -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0xDC -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0xDD -> BOX DRAWINGS LIGHT VERTICAL + '\u03c0', // π 0xDE -> GREEK SMALL LETTER PI + '\u25e5', // ◥ 0xDF -> BLACK UPPER RIGHT TRIANGLE + '\u00a0', // 0xE0 -> NO-BREAK SPACE + '\u258c', // ▌ 0xE1 -> LEFT HALF BLOCK + '\u2584', // ▄ 0xE2 -> LOWER HALF BLOCK + '\u2594', // ▔ 0xE3 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0xE4 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0xE5 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0xE6 -> MEDIUM SHADE + '\u2595', // ▕ 0xE7 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0xE8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\u25e4', // ◤ 0xE9 -> BLACK UPPER LEFT TRIANGLE + '\uf130', //  0xEA -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0xEB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0xEC -> QUADRANT LOWER RIGHT + '\u2514', // └ 0xED -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0xEE -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0xEF -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0xF0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0xF1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0xF2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0xF3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0xF4 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0xF5 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0xF6 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0xF7 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0xF8 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0xF9 -> LOWER THREE EIGHTHS BLOCK + '\uf12d', //  0xFA -> ONE EIGHTH BLOCK UP AND LEFT (CUS) + '\u2596', // ▖ 0xFB -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0xFC -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0xFD -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0xFE -> QUADRANT UPPER LEFT + '\u03c0' // π 0xFF -> GREEK SMALL LETTER PI + ) + + private val decodingScreencodeLowercase = arrayOf( + '@' , // @ 0x00 -> COMMERCIAL AT + 'a' , // a 0x01 -> LATIN SMALL LETTER A + 'b' , // b 0x02 -> LATIN SMALL LETTER B + 'c' , // c 0x03 -> LATIN SMALL LETTER C + 'd' , // d 0x04 -> LATIN SMALL LETTER D + 'e' , // e 0x05 -> LATIN SMALL LETTER E + 'f' , // f 0x06 -> LATIN SMALL LETTER F + 'g' , // g 0x07 -> LATIN SMALL LETTER G + 'h' , // h 0x08 -> LATIN SMALL LETTER H + 'i' , // i 0x09 -> LATIN SMALL LETTER I + 'j' , // j 0x0A -> LATIN SMALL LETTER J + 'k' , // k 0x0B -> LATIN SMALL LETTER K + 'l' , // l 0x0C -> LATIN SMALL LETTER L + 'm' , // m 0x0D -> LATIN SMALL LETTER M + 'n' , // n 0x0E -> LATIN SMALL LETTER N + 'o' , // o 0x0F -> LATIN SMALL LETTER O + 'p' , // p 0x10 -> LATIN SMALL LETTER P + 'q' , // q 0x11 -> LATIN SMALL LETTER Q + 'r' , // r 0x12 -> LATIN SMALL LETTER R + 's' , // s 0x13 -> LATIN SMALL LETTER S + 't' , // t 0x14 -> LATIN SMALL LETTER T + 'u' , // u 0x15 -> LATIN SMALL LETTER U + 'v' , // v 0x16 -> LATIN SMALL LETTER V + 'w' , // w 0x17 -> LATIN SMALL LETTER W + 'x' , // x 0x18 -> LATIN SMALL LETTER X + 'y' , // y 0x19 -> LATIN SMALL LETTER Y + 'z' , // z 0x1A -> LATIN SMALL LETTER Z + '[' , // [ 0x1B -> LEFT SQUARE BRACKET + '\u00a3', // £ 0x1C -> POUND SIGN + ']' , // ] 0x1D -> RIGHT SQUARE BRACKET + '\u2191', // ↑ 0x1E -> UPWARDS ARROW + '\u2190', // ← 0x1F -> LEFTWARDS ARROW + ' ' , // 0x20 -> SPACE + '!' , // ! 0x21 -> EXCLAMATION MARK + '"' , // " 0x22 -> QUOTATION MARK + '#' , // # 0x23 -> NUMBER SIGN + '$' , // $ 0x24 -> DOLLAR SIGN + '%' , // % 0x25 -> PERCENT SIGN + '&' , // & 0x26 -> AMPERSAND + '\'' , // ' 0x27 -> APOSTROPHE + '(' , // ( 0x28 -> LEFT PARENTHESIS + ')' , // ) 0x29 -> RIGHT PARENTHESIS + '*' , // * 0x2A -> ASTERISK + '+' , // + 0x2B -> PLUS SIGN + ',' , // , 0x2C -> COMMA + '-' , // - 0x2D -> HYPHEN-MINUS + '.' , // . 0x2E -> FULL STOP + '/' , // / 0x2F -> SOLIDUS + '0' , // 0 0x30 -> DIGIT ZERO + '1' , // 1 0x31 -> DIGIT ONE + '2' , // 2 0x32 -> DIGIT TWO + '3' , // 3 0x33 -> DIGIT THREE + '4' , // 4 0x34 -> DIGIT FOUR + '5' , // 5 0x35 -> DIGIT FIVE + '6' , // 6 0x36 -> DIGIT SIX + '7' , // 7 0x37 -> DIGIT SEVEN + '8' , // 8 0x38 -> DIGIT EIGHT + '9' , // 9 0x39 -> DIGIT NINE + ':' , // : 0x3A -> COLON + ';' , // ; 0x3B -> SEMICOLON + '<' , // < 0x3C -> LESS-THAN SIGN + '=' , // = 0x3D -> EQUALS SIGN + '>' , // > 0x3E -> GREATER-THAN SIGN + '?' , // ? 0x3F -> QUESTION MARK + '\u2500', // ─ 0x40 -> BOX DRAWINGS LIGHT HORIZONTAL + 'A' , // A 0x41 -> LATIN CAPITAL LETTER A + 'B' , // B 0x42 -> LATIN CAPITAL LETTER B + 'C' , // C 0x43 -> LATIN CAPITAL LETTER C + 'D' , // D 0x44 -> LATIN CAPITAL LETTER D + 'E' , // E 0x45 -> LATIN CAPITAL LETTER E + 'F' , // F 0x46 -> LATIN CAPITAL LETTER F + 'G' , // G 0x47 -> LATIN CAPITAL LETTER G + 'H' , // H 0x48 -> LATIN CAPITAL LETTER H + 'I' , // I 0x49 -> LATIN CAPITAL LETTER I + 'J' , // J 0x4A -> LATIN CAPITAL LETTER J + 'K' , // K 0x4B -> LATIN CAPITAL LETTER K + 'L' , // L 0x4C -> LATIN CAPITAL LETTER L + 'M' , // M 0x4D -> LATIN CAPITAL LETTER M + 'N' , // N 0x4E -> LATIN CAPITAL LETTER N + 'O' , // O 0x4F -> LATIN CAPITAL LETTER O + 'P' , // P 0x50 -> LATIN CAPITAL LETTER P + 'Q' , // Q 0x51 -> LATIN CAPITAL LETTER Q + 'R' , // R 0x52 -> LATIN CAPITAL LETTER R + 'S' , // S 0x53 -> LATIN CAPITAL LETTER S + 'T' , // T 0x54 -> LATIN CAPITAL LETTER T + 'U' , // U 0x55 -> LATIN CAPITAL LETTER U + 'V' , // V 0x56 -> LATIN CAPITAL LETTER V + 'W' , // W 0x57 -> LATIN CAPITAL LETTER W + 'X' , // X 0x58 -> LATIN CAPITAL LETTER X + 'Y' , // Y 0x59 -> LATIN CAPITAL LETTER Y + 'Z' , // Z 0x5A -> LATIN CAPITAL LETTER Z + '\u253c', // ┼ 0x5B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0x5C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0x5D -> BOX DRAWINGS LIGHT VERTICAL + '\u2592', // ▒ 0x5E -> MEDIUM SHADE + '\uf139', //  0x5F -> MEDIUM SHADE SLASHED LEFT (CUS) + '\u00a0', // 0x60 -> NO-BREAK SPACE + '\u258c', // ▌ 0x61 -> LEFT HALF BLOCK + '\u2584', // ▄ 0x62 -> LOWER HALF BLOCK + '\u2594', // ▔ 0x63 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0x64 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0x65 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0x66 -> MEDIUM SHADE + '\u2595', // ▕ 0x67 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0x68 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\uf13a', //  0x69 -> MEDIUM SHADE SLASHED RIGHT (CUS) + '\uf130', //  0x6A -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0x6B -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0x6C -> QUADRANT LOWER RIGHT + '\u2514', // └ 0x6D -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0x6E -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0x6F -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0x70 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0x71 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0x72 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0x73 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0x74 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0x75 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0x76 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0x77 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0x78 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0x79 -> LOWER THREE EIGHTHS BLOCK + '\u2713', // ✓ 0x7A -> CHECK MARK + '\u2596', // ▖ 0x7B -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0x7C -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0x7D -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0x7E -> QUADRANT UPPER LEFT + '\u259a', // ▚ 0x7F -> QUADRANT UPPER LEFT AND LOWER RIGHT + '\ufffe', // 0x80 -> UNDEFINED + '\ufffe', // 0x81 -> UNDEFINED + '\ufffe', // 0x82 -> UNDEFINED + '\ufffe', // 0x83 -> UNDEFINED + '\ufffe', // 0x84 -> UNDEFINED + '\ufffe', // 0x85 -> UNDEFINED + '\ufffe', // 0x86 -> UNDEFINED + '\ufffe', // 0x87 -> UNDEFINED + '\ufffe', // 0x88 -> UNDEFINED + '\ufffe', // 0x89 -> UNDEFINED + '\ufffe', // 0x8A -> UNDEFINED + '\ufffe', // 0x8B -> UNDEFINED + '\ufffe', // 0x8C -> UNDEFINED + '\ufffe', // 0x8D -> UNDEFINED + '\ufffe', // 0x8E -> UNDEFINED + '\ufffe', // 0x8F -> UNDEFINED + '\ufffe', // 0x90 -> UNDEFINED + '\ufffe', // 0x91 -> UNDEFINED + '\ufffe', // 0x92 -> UNDEFINED + '\ufffe', // 0x93 -> UNDEFINED + '\ufffe', // 0x94 -> UNDEFINED + '\ufffe', // 0x95 -> UNDEFINED + '\ufffe', // 0x96 -> UNDEFINED + '\ufffe', // 0x97 -> UNDEFINED + '\ufffe', // 0x98 -> UNDEFINED + '\ufffe', // 0x99 -> UNDEFINED + '\ufffe', // 0x9A -> UNDEFINED + '\ufffe', // 0x9B -> UNDEFINED + '\ufffe', // 0x9C -> UNDEFINED + '\ufffe', // 0x9D -> UNDEFINED + '\ufffe', // 0x9E -> UNDEFINED + '\ufffe', // 0x9F -> UNDEFINED + '\ufffe', // 0xA0 -> UNDEFINED + '\ufffe', // 0xA1 -> UNDEFINED + '\ufffe', // 0xA2 -> UNDEFINED + '\ufffe', // 0xA3 -> UNDEFINED + '\ufffe', // 0xA4 -> UNDEFINED + '\ufffe', // 0xA5 -> UNDEFINED + '\ufffe', // 0xA6 -> UNDEFINED + '\ufffe', // 0xA7 -> UNDEFINED + '\ufffe', // 0xA8 -> UNDEFINED + '\ufffe', // 0xA9 -> UNDEFINED + '\ufffe', // 0xAA -> UNDEFINED + '\ufffe', // 0xAB -> UNDEFINED + '\ufffe', // 0xAC -> UNDEFINED + '\ufffe', // 0xAD -> UNDEFINED + '\ufffe', // 0xAE -> UNDEFINED + '\ufffe', // 0xAF -> UNDEFINED + '\ufffe', // 0xB0 -> UNDEFINED + '\ufffe', // 0xB1 -> UNDEFINED + '\ufffe', // 0xB2 -> UNDEFINED + '\ufffe', // 0xB3 -> UNDEFINED + '\ufffe', // 0xB4 -> UNDEFINED + '\ufffe', // 0xB5 -> UNDEFINED + '\ufffe', // 0xB6 -> UNDEFINED + '\ufffe', // 0xB7 -> UNDEFINED + '\ufffe', // 0xB8 -> UNDEFINED + '\ufffe', // 0xB9 -> UNDEFINED + '\ufffe', // 0xBA -> UNDEFINED + '\ufffe', // 0xBB -> UNDEFINED + '\ufffe', // 0xBC -> UNDEFINED + '\ufffe', // 0xBD -> UNDEFINED + '\ufffe', // 0xBE -> UNDEFINED + '\ufffe', // 0xBF -> UNDEFINED + '\ufffe', // 0xC0 -> UNDEFINED + '\ufffe', // 0xC1 -> UNDEFINED + '\ufffe', // 0xC2 -> UNDEFINED + '\ufffe', // 0xC3 -> UNDEFINED + '\ufffe', // 0xC4 -> UNDEFINED + '\ufffe', // 0xC5 -> UNDEFINED + '\ufffe', // 0xC6 -> UNDEFINED + '\ufffe', // 0xC7 -> UNDEFINED + '\ufffe', // 0xC8 -> UNDEFINED + '\ufffe', // 0xC9 -> UNDEFINED + '\ufffe', // 0xCA -> UNDEFINED + '\ufffe', // 0xCB -> UNDEFINED + '\ufffe', // 0xCC -> UNDEFINED + '\ufffe', // 0xCD -> UNDEFINED + '\ufffe', // 0xCE -> UNDEFINED + '\ufffe', // 0xCF -> UNDEFINED + '\ufffe', // 0xD0 -> UNDEFINED + '\ufffe', // 0xD1 -> UNDEFINED + '\ufffe', // 0xD2 -> UNDEFINED + '\ufffe', // 0xD3 -> UNDEFINED + '\ufffe', // 0xD4 -> UNDEFINED + '\ufffe', // 0xD5 -> UNDEFINED + '\ufffe', // 0xD6 -> UNDEFINED + '\ufffe', // 0xD7 -> UNDEFINED + '\ufffe', // 0xD8 -> UNDEFINED + '\ufffe', // 0xD9 -> UNDEFINED + '\ufffe', // 0xDA -> UNDEFINED + '\ufffe', // 0xDB -> UNDEFINED + '\ufffe', // 0xDC -> UNDEFINED + '\ufffe', // 0xDD -> UNDEFINED + '\ufffe', // 0xDE -> UNDEFINED + '\ufffe', // 0xDF -> UNDEFINED + '\ufffe', // 0xE0 -> UNDEFINED + '\ufffe', // 0xE1 -> UNDEFINED + '\ufffe', // 0xE2 -> UNDEFINED + '\ufffe', // 0xE3 -> UNDEFINED + '\ufffe', // 0xE4 -> UNDEFINED + '\ufffe', // 0xE5 -> UNDEFINED + '\ufffe', // 0xE6 -> UNDEFINED + '\ufffe', // 0xE7 -> UNDEFINED + '\ufffe', // 0xE8 -> UNDEFINED + '\ufffe', // 0xE9 -> UNDEFINED + '\ufffe', // 0xEA -> UNDEFINED + '\ufffe', // 0xEB -> UNDEFINED + '\ufffe', // 0xEC -> UNDEFINED + '\ufffe', // 0xED -> UNDEFINED + '\ufffe', // 0xEE -> UNDEFINED + '\ufffe', // 0xEF -> UNDEFINED + '\ufffe', // 0xF0 -> UNDEFINED + '\ufffe', // 0xF1 -> UNDEFINED + '\ufffe', // 0xF2 -> UNDEFINED + '\ufffe', // 0xF3 -> UNDEFINED + '\ufffe', // 0xF4 -> UNDEFINED + '\ufffe', // 0xF5 -> UNDEFINED + '\ufffe', // 0xF6 -> UNDEFINED + '\ufffe', // 0xF7 -> UNDEFINED + '\ufffe', // 0xF8 -> UNDEFINED + '\ufffe', // 0xF9 -> UNDEFINED + '\ufffe', // 0xFA -> UNDEFINED + '\ufffe', // 0xFB -> UNDEFINED + '\ufffe', // 0xFC -> UNDEFINED + '\ufffe', // 0xFD -> UNDEFINED + '\ufffe', // 0xFE -> UNDEFINED + '\ufffe' // 0xFF -> UNDEFINED + ) + + private val decodingScreencodeUppercase = arrayOf( + '@' , // @ 0x00 -> COMMERCIAL AT + 'A' , // A 0x01 -> LATIN CAPITAL LETTER A + 'B' , // B 0x02 -> LATIN CAPITAL LETTER B + 'C' , // C 0x03 -> LATIN CAPITAL LETTER C + 'D' , // D 0x04 -> LATIN CAPITAL LETTER D + 'E' , // E 0x05 -> LATIN CAPITAL LETTER E + 'F' , // F 0x06 -> LATIN CAPITAL LETTER F + 'G' , // G 0x07 -> LATIN CAPITAL LETTER G + 'H' , // H 0x08 -> LATIN CAPITAL LETTER H + 'I' , // I 0x09 -> LATIN CAPITAL LETTER I + 'J' , // J 0x0A -> LATIN CAPITAL LETTER J + 'K' , // K 0x0B -> LATIN CAPITAL LETTER K + 'L' , // L 0x0C -> LATIN CAPITAL LETTER L + 'M' , // M 0x0D -> LATIN CAPITAL LETTER M + 'N' , // N 0x0E -> LATIN CAPITAL LETTER N + 'O' , // O 0x0F -> LATIN CAPITAL LETTER O + 'P' , // P 0x10 -> LATIN CAPITAL LETTER P + 'Q' , // Q 0x11 -> LATIN CAPITAL LETTER Q + 'R' , // R 0x12 -> LATIN CAPITAL LETTER R + 'S' , // S 0x13 -> LATIN CAPITAL LETTER S + 'T' , // T 0x14 -> LATIN CAPITAL LETTER T + 'U' , // U 0x15 -> LATIN CAPITAL LETTER U + 'V' , // V 0x16 -> LATIN CAPITAL LETTER V + 'W' , // W 0x17 -> LATIN CAPITAL LETTER W + 'X' , // X 0x18 -> LATIN CAPITAL LETTER X + 'Y' , // Y 0x19 -> LATIN CAPITAL LETTER Y + 'Z' , // Z 0x1A -> LATIN CAPITAL LETTER Z + '[' , // [ 0x1B -> LEFT SQUARE BRACKET + '\u00a3', // £ 0x1C -> POUND SIGN + ']' , // ] 0x1D -> RIGHT SQUARE BRACKET + '\u2191', // ↑ 0x1E -> UPWARDS ARROW + '\u2190', // ← 0x1F -> LEFTWARDS ARROW + ' ' , // 0x20 -> SPACE + '!' , // ! 0x21 -> EXCLAMATION MARK + '"' , // " 0x22 -> QUOTATION MARK + '#' , // # 0x23 -> NUMBER SIGN + '$' , // $ 0x24 -> DOLLAR SIGN + '%' , // % 0x25 -> PERCENT SIGN + '&' , // & 0x26 -> AMPERSAND + '\'' , // ' 0x27 -> APOSTROPHE + '(' , // ( 0x28 -> LEFT PARENTHESIS + ')' , // ) 0x29 -> RIGHT PARENTHESIS + '*' , // * 0x2A -> ASTERISK + '+' , // + 0x2B -> PLUS SIGN + ',' , // , 0x2C -> COMMA + '-' , // - 0x2D -> HYPHEN-MINUS + '.' , // . 0x2E -> FULL STOP + '/' , // / 0x2F -> SOLIDUS + '0' , // 0 0x30 -> DIGIT ZERO + '1' , // 1 0x31 -> DIGIT ONE + '2' , // 2 0x32 -> DIGIT TWO + '3' , // 3 0x33 -> DIGIT THREE + '4' , // 4 0x34 -> DIGIT FOUR + '5' , // 5 0x35 -> DIGIT FIVE + '6' , // 6 0x36 -> DIGIT SIX + '7' , // 7 0x37 -> DIGIT SEVEN + '8' , // 8 0x38 -> DIGIT EIGHT + '9' , // 9 0x39 -> DIGIT NINE + ':' , // : 0x3A -> COLON + ';' , // ; 0x3B -> SEMICOLON + '<' , // < 0x3C -> LESS-THAN SIGN + '=' , // = 0x3D -> EQUALS SIGN + '>' , // > 0x3E -> GREATER-THAN SIGN + '?' , // ? 0x3F -> QUESTION MARK + '\u2500', // ─ 0x40 -> BOX DRAWINGS LIGHT HORIZONTAL + '\u2660', // ♠ 0x41 -> BLACK SPADE SUIT + '\u2502', // │ 0x42 -> BOX DRAWINGS LIGHT VERTICAL + '\u2500', // ─ 0x43 -> BOX DRAWINGS LIGHT HORIZONTAL + '\uf122', //  0x44 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) + '\uf123', //  0x45 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) + '\uf124', //  0x46 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) + '\uf126', //  0x47 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) + '\uf128', //  0x48 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) + '\u256e', // ╮ 0x49 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT + '\u2570', // ╰ 0x4A -> BOX DRAWINGS LIGHT ARC UP AND RIGHT + '\u256f', // ╯ 0x4B -> BOX DRAWINGS LIGHT ARC UP AND LEFT + '\uf12a', //  0x4C -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) + '\u2572', // ╲ 0x4D -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + '\u2571', // ╱ 0x4E -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + '\uf12b', //  0x4F -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) + '\uf12c', //  0x50 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) + '\u25cf', // ● 0x51 -> BLACK CIRCLE + '\uf125', //  0x52 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) + '\u2665', // ♥ 0x53 -> BLACK HEART SUIT + '\uf127', //  0x54 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) + '\u256d', // ╭ 0x55 -> BOX DRAWINGS LIGHT ARC DOWN AND RIGHT + '\u2573', // ╳ 0x56 -> BOX DRAWINGS LIGHT DIAGONAL CROSS + '\u25cb', // ○ 0x57 -> WHITE CIRCLE + '\u2663', // ♣ 0x58 -> BLACK CLUB SUIT + '\uf129', //  0x59 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) + '\u2666', // ♦ 0x5A -> BLACK DIAMOND SUIT + '\u253c', // ┼ 0x5B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + '\uf12e', //  0x5C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) + '\u2502', // │ 0x5D -> BOX DRAWINGS LIGHT VERTICAL + '\u03c0', // π 0x5E -> GREEK SMALL LETTER PI + '\u25e5', // ◥ 0x5F -> BLACK UPPER RIGHT TRIANGLE + '\u00a0', // 0x60 -> NO-BREAK SPACE + '\u258c', // ▌ 0x61 -> LEFT HALF BLOCK + '\u2584', // ▄ 0x62 -> LOWER HALF BLOCK + '\u2594', // ▔ 0x63 -> UPPER ONE EIGHTH BLOCK + '\u2581', // ▁ 0x64 -> LOWER ONE EIGHTH BLOCK + '\u258f', // ▏ 0x65 -> LEFT ONE EIGHTH BLOCK + '\u2592', // ▒ 0x66 -> MEDIUM SHADE + '\u2595', // ▕ 0x67 -> RIGHT ONE EIGHTH BLOCK + '\uf12f', //  0x68 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) + '\u25e4', // ◤ 0x69 -> BLACK UPPER LEFT TRIANGLE + '\uf130', //  0x6A -> RIGHT ONE QUARTER BLOCK (CUS) + '\u251c', // ├ 0x6B -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT + '\u2597', // ▗ 0x6C -> QUADRANT LOWER RIGHT + '\u2514', // └ 0x6D -> BOX DRAWINGS LIGHT UP AND RIGHT + '\u2510', // ┐ 0x6E -> BOX DRAWINGS LIGHT DOWN AND LEFT + '\u2582', // ▂ 0x6F -> LOWER ONE QUARTER BLOCK + '\u250c', // ┌ 0x70 -> BOX DRAWINGS LIGHT DOWN AND RIGHT + '\u2534', // ┴ 0x71 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL + '\u252c', // ┬ 0x72 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + '\u2524', // ┤ 0x73 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT + '\u258e', // ▎ 0x74 -> LEFT ONE QUARTER BLOCK + '\u258d', // ▍ 0x75 -> LEFT THREE EIGTHS BLOCK + '\uf131', //  0x76 -> RIGHT THREE EIGHTHS BLOCK (CUS) + '\uf132', //  0x77 -> UPPER ONE QUARTER BLOCK (CUS) + '\uf133', //  0x78 -> UPPER THREE EIGHTS BLOCK (CUS) + '\u2583', // ▃ 0x79 -> LOWER THREE EIGHTHS BLOCK + '\uf12d', //  0x7A -> ONE EIGHTH BLOCK UP AND LEFT (CUS) + '\u2596', // ▖ 0x7B -> QUADRANT LOWER LEFT + '\u259d', // ▝ 0x7C -> QUADRANT UPPER RIGHT + '\u2518', // ┘ 0x7D -> BOX DRAWINGS LIGHT UP AND LEFT + '\u2598', // ▘ 0x7E -> QUADRANT UPPER LEFT + '\u259a', // ▚ 0x7F -> QUADRANT UPPER LEFT AND LOWER RIGHT + '\ufffe', // 0x80 -> UNDEFINED + '\ufffe', // 0x81 -> UNDEFINED + '\ufffe', // 0x82 -> UNDEFINED + '\ufffe', // 0x83 -> UNDEFINED + '\ufffe', // 0x84 -> UNDEFINED + '\ufffe', // 0x85 -> UNDEFINED + '\ufffe', // 0x86 -> UNDEFINED + '\ufffe', // 0x87 -> UNDEFINED + '\ufffe', // 0x88 -> UNDEFINED + '\ufffe', // 0x89 -> UNDEFINED + '\ufffe', // 0x8A -> UNDEFINED + '\ufffe', // 0x8B -> UNDEFINED + '\ufffe', // 0x8C -> UNDEFINED + '\ufffe', // 0x8D -> UNDEFINED + '\ufffe', // 0x8E -> UNDEFINED + '\ufffe', // 0x8F -> UNDEFINED + '\ufffe', // 0x90 -> UNDEFINED + '\ufffe', // 0x91 -> UNDEFINED + '\ufffe', // 0x92 -> UNDEFINED + '\ufffe', // 0x93 -> UNDEFINED + '\ufffe', // 0x94 -> UNDEFINED + '\ufffe', // 0x95 -> UNDEFINED + '\ufffe', // 0x96 -> UNDEFINED + '\ufffe', // 0x97 -> UNDEFINED + '\ufffe', // 0x98 -> UNDEFINED + '\ufffe', // 0x99 -> UNDEFINED + '\ufffe', // 0x9A -> UNDEFINED + '\ufffe', // 0x9B -> UNDEFINED + '\ufffe', // 0x9C -> UNDEFINED + '\ufffe', // 0x9D -> UNDEFINED + '\ufffe', // 0x9E -> UNDEFINED + '\ufffe', // 0x9F -> UNDEFINED + '\ufffe', // 0xA0 -> UNDEFINED + '\ufffe', // 0xA1 -> UNDEFINED + '\ufffe', // 0xA2 -> UNDEFINED + '\ufffe', // 0xA3 -> UNDEFINED + '\ufffe', // 0xA4 -> UNDEFINED + '\ufffe', // 0xA5 -> UNDEFINED + '\ufffe', // 0xA6 -> UNDEFINED + '\ufffe', // 0xA7 -> UNDEFINED + '\ufffe', // 0xA8 -> UNDEFINED + '\ufffe', // 0xA9 -> UNDEFINED + '\ufffe', // 0xAA -> UNDEFINED + '\ufffe', // 0xAB -> UNDEFINED + '\ufffe', // 0xAC -> UNDEFINED + '\ufffe', // 0xAD -> UNDEFINED + '\ufffe', // 0xAE -> UNDEFINED + '\ufffe', // 0xAF -> UNDEFINED + '\ufffe', // 0xB0 -> UNDEFINED + '\ufffe', // 0xB1 -> UNDEFINED + '\ufffe', // 0xB2 -> UNDEFINED + '\ufffe', // 0xB3 -> UNDEFINED + '\ufffe', // 0xB4 -> UNDEFINED + '\ufffe', // 0xB5 -> UNDEFINED + '\ufffe', // 0xB6 -> UNDEFINED + '\ufffe', // 0xB7 -> UNDEFINED + '\ufffe', // 0xB8 -> UNDEFINED + '\ufffe', // 0xB9 -> UNDEFINED + '\ufffe', // 0xBA -> UNDEFINED + '\ufffe', // 0xBB -> UNDEFINED + '\ufffe', // 0xBC -> UNDEFINED + '\ufffe', // 0xBD -> UNDEFINED + '\ufffe', // 0xBE -> UNDEFINED + '\ufffe', // 0xBF -> UNDEFINED + '\ufffe', // 0xC0 -> UNDEFINED + '\ufffe', // 0xC1 -> UNDEFINED + '\ufffe', // 0xC2 -> UNDEFINED + '\ufffe', // 0xC3 -> UNDEFINED + '\ufffe', // 0xC4 -> UNDEFINED + '\ufffe', // 0xC5 -> UNDEFINED + '\ufffe', // 0xC6 -> UNDEFINED + '\ufffe', // 0xC7 -> UNDEFINED + '\ufffe', // 0xC8 -> UNDEFINED + '\ufffe', // 0xC9 -> UNDEFINED + '\ufffe', // 0xCA -> UNDEFINED + '\ufffe', // 0xCB -> UNDEFINED + '\ufffe', // 0xCC -> UNDEFINED + '\ufffe', // 0xCD -> UNDEFINED + '\ufffe', // 0xCE -> UNDEFINED + '\ufffe', // 0xCF -> UNDEFINED + '\ufffe', // 0xD0 -> UNDEFINED + '\ufffe', // 0xD1 -> UNDEFINED + '\ufffe', // 0xD2 -> UNDEFINED + '\ufffe', // 0xD3 -> UNDEFINED + '\ufffe', // 0xD4 -> UNDEFINED + '\ufffe', // 0xD5 -> UNDEFINED + '\ufffe', // 0xD6 -> UNDEFINED + '\ufffe', // 0xD7 -> UNDEFINED + '\ufffe', // 0xD8 -> UNDEFINED + '\ufffe', // 0xD9 -> UNDEFINED + '\ufffe', // 0xDA -> UNDEFINED + '\ufffe', // 0xDB -> UNDEFINED + '\ufffe', // 0xDC -> UNDEFINED + '\ufffe', // 0xDD -> UNDEFINED + '\ufffe', // 0xDE -> UNDEFINED + '\ufffe', // 0xDF -> UNDEFINED + '\ufffe', // 0xE0 -> UNDEFINED + '\ufffe', // 0xE1 -> UNDEFINED + '\ufffe', // 0xE2 -> UNDEFINED + '\ufffe', // 0xE3 -> UNDEFINED + '\ufffe', // 0xE4 -> UNDEFINED + '\ufffe', // 0xE5 -> UNDEFINED + '\ufffe', // 0xE6 -> UNDEFINED + '\ufffe', // 0xE7 -> UNDEFINED + '\ufffe', // 0xE8 -> UNDEFINED + '\ufffe', // 0xE9 -> UNDEFINED + '\ufffe', // 0xEA -> UNDEFINED + '\ufffe', // 0xEB -> UNDEFINED + '\ufffe', // 0xEC -> UNDEFINED + '\ufffe', // 0xED -> UNDEFINED + '\ufffe', // 0xEE -> UNDEFINED + '\ufffe', // 0xEF -> UNDEFINED + '\ufffe', // 0xF0 -> UNDEFINED + '\ufffe', // 0xF1 -> UNDEFINED + '\ufffe', // 0xF2 -> UNDEFINED + '\ufffe', // 0xF3 -> UNDEFINED + '\ufffe', // 0xF4 -> UNDEFINED + '\ufffe', // 0xF5 -> UNDEFINED + '\ufffe', // 0xF6 -> UNDEFINED + '\ufffe', // 0xF7 -> UNDEFINED + '\ufffe', // 0xF8 -> UNDEFINED + '\ufffe', // 0xF9 -> UNDEFINED + '\ufffe', // 0xFA -> UNDEFINED + '\ufffe', // 0xFB -> UNDEFINED + '\ufffe', // 0xFC -> UNDEFINED + '\ufffe', // 0xFD -> UNDEFINED + '\ufffe', // 0xFE -> UNDEFINED + '\ufffe' // 0xFF -> UNDEFINED + ) + + // encoding: from unicode to Petscii/Screencodes (0-255) + private val encodingPetsciiLowercase = decodingPetsciiLowercase.withIndex().associate{it.value to it.index} + private val encodingPetsciiUppercase = decodingPetsciiUppercase.withIndex().associate{it.value to it.index} + private val encodingScreencodeLowercase = decodingScreencodeLowercase.withIndex().associate{it.value to it.index} + private val encodingScreencodeUppercase = decodingScreencodeUppercase.withIndex().associate{it.value to it.index} + + private fun replaceSpecial(chr: Char): Char = + // characters often used in C like source code can be translated with a little bit of fantasy: + when(chr) { + '^' -> '↑' + '_' -> '▁' + '{' -> '┤' + '}' -> '├' + '|' -> '│' + '\\' -> '╲' + else -> chr + } + + fun encodePetscii(text: String, lowercase: Boolean = false): List { + fun encodeChar(chr3: Char, lowercase: Boolean): Short { + val chr = replaceSpecial(chr3) + val screencode = if(lowercase) encodingPetsciiLowercase[chr] else encodingPetsciiUppercase[chr] + return screencode?.toShort() ?: when (chr) { + '\u0000' -> 0.toShort() + in '\u8000'..'\u80ff' -> { + // special case: take the lower 8 bit hex value directly + (chr.code - 0x8000).toShort() + } + else -> { + val case = if (lowercase) "lower" else "upper" + throw CharConversionException("no ${case}Petscii character for '${escape(chr.toString())}' (${chr.code})") + } + } + } + + return text.map{ + try { + encodeChar(it, lowercase) + } catch (x: CharConversionException) { + encodeChar(it, !lowercase) + } + } + } + + fun decodePetscii(petscii: Iterable, lowercase: Boolean = false): String { + return petscii.map { + val code = it.toInt() + try { + if(lowercase) decodingPetsciiLowercase[code] else decodingPetsciiUppercase[code] + } catch(x: CharConversionException) { + if(lowercase) decodingPetsciiUppercase[code] else decodingPetsciiLowercase[code] + } + }.joinToString("") + } + + fun encodeScreencode(text: String, lowercase: Boolean = false): List { + fun encodeChar(chr3: Char, lowercase: Boolean): Short { + val chr = replaceSpecial(chr3) + val screencode = if(lowercase) encodingScreencodeLowercase[chr] else encodingScreencodeUppercase[chr] + return screencode?.toShort() ?: when (chr) { + '\u0000' -> 0.toShort() + in '\u8000'..'\u80ff' -> { + // special case: take the lower 8 bit hex value directly + (chr.code - 0x8000).toShort() + } + else -> { + val case = if (lowercase) "lower" else "upper" + throw CharConversionException("no ${case}Screencode character for '${escape(chr.toString())}' (${chr.code})") + } + } + } + + return text.map{ + try { + encodeChar(it, lowercase) + } catch (x: CharConversionException) { + encodeChar(it, !lowercase) + } + } + } + + fun decodeScreencode(screencode: Iterable, lowercase: Boolean = false): String { + return screencode.map { + val code = it.toInt() + try { + if (lowercase) decodingScreencodeLowercase[code] else decodingScreencodeUppercase[code] + } catch (x: CharConversionException) { + if (lowercase) decodingScreencodeUppercase[code] else decodingScreencodeLowercase[code] + } + }.joinToString("") + } + + fun petscii2scr(petscii_code: Short, inverseVideo: Boolean): Short { + val code = when { + petscii_code <= 0x1f -> petscii_code + 128 + petscii_code <= 0x3f -> petscii_code.toInt() + petscii_code <= 0x5f -> petscii_code - 64 + petscii_code <= 0x7f -> petscii_code - 32 + petscii_code <= 0x9f -> petscii_code + 64 + petscii_code <= 0xbf -> petscii_code - 64 + petscii_code <= 0xfe -> petscii_code - 128 + petscii_code == 255.toShort() -> 95 + else -> throw CharConversionException("petscii code out of range") + } + if(inverseVideo) + return (code or 0x80).toShort() + return code.toShort() + } + + fun scr2petscii(screencode: Short): Short { + val petscii = when { + screencode <= 0x1f -> screencode + 64 + screencode <= 0x3f -> screencode.toInt() + screencode <= 0x5d -> screencode +123 + screencode == 0x5e.toShort() -> 255 + screencode == 0x5f.toShort() -> 223 + screencode <= 0x7f -> screencode + 64 + screencode <= 0xbf -> screencode - 128 + screencode <= 0xfe -> screencode - 64 + screencode == 255.toShort() -> 191 + else -> throw CharConversionException("screencode out of range") + } + return petscii.toShort() + } +} diff --git a/compilerAst/src/prog8/parser/PetsciiEncoding.kt b/compilerAst/src/prog8/parser/PetsciiEncoding.kt new file mode 100644 index 000000000..58f752834 --- /dev/null +++ b/compilerAst/src/prog8/parser/PetsciiEncoding.kt @@ -0,0 +1,24 @@ +package prog8.parser + +import prog8.ast.IStringEncoding +import java.io.CharConversionException + + +/** + * TODO: remove once [IStringEncoding] has been to compiler module + */ +object PetsciiEncoding : IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean) = + try { + if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true) + } catch (x: CharConversionException) { + throw CharConversionException("can't convert string to target machine's char encoding: ${x.message}") + } + + override fun decodeString(bytes: List, altEncoding: Boolean) = + try { + if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true) + } catch (x: CharConversionException) { + throw CharConversionException("can't decode string: ${x.message}") + } +} diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 51b60ef44..5a7a4dfc6 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -4,20 +4,8 @@ import org.antlr.v4.runtime.* import org.antlr.v4.runtime.misc.ParseCancellationException import prog8.ast.antlr.toAst import prog8.ast.Module -import prog8.ast.base.Position -import prog8.ast.IStringEncoding -object DummyEncoding: IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("move StringEncoding out of compilerAst") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("move StringEncoding out of compilerAst") - } -} - class Prog8ErrorStrategy: BailErrorStrategy() { override fun recover(recognizer: Parser?, e: RecognitionException?) { try { @@ -56,10 +44,14 @@ class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorList parser.addErrorListener(errorListener) val parseTree = parser.module() + val moduleName = "anonymous" + val module = parseTree.toAst(moduleName, pathFrom(""), PetsciiEncoding) // TODO: use Module ctor directly - val moduleAst = parseTree.toAst("anonymous", pathFrom(""), DummyEncoding) - return moduleAst + for (statement in module.statements) { + statement.linkParents(module) + } + return module } } diff --git a/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt b/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt new file mode 100644 index 000000000..674c3d77e --- /dev/null +++ b/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt @@ -0,0 +1,16 @@ +package prog8.parser + +import prog8.ast.IStringEncoding + +/** + * TODO: remove once [IStringEncoding] has been to compiler module + */ +object ThrowTodoEncoding: IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("move StringEncoding out of compilerAst") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("move StringEncoding out of compilerAst") + } +} From 99b1cec2e1620aba89e1d89e8b5b31b1798d2bca Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 18:10:26 +0200 Subject: [PATCH 03/68] */+ move ParsingFailedError to Prog8Parser.kt, intro ParseError (soon to replace ParsingFailedError), start testing proper error location info --- compilerAst/src/prog8/parser/ModuleParsing.kt | 3 +- compilerAst/src/prog8/parser/Prog8Parser.kt | 93 +++++++--- compilerAst/test/TestAntlrParser.kt | 164 ++++++------------ compilerAst/test/TestModuleImporter.kt | 81 +++++++++ .../test/fixtures/file_with_syntax_error.p8 | 2 + .../fixtures/import_file_with_syntax_error.p8 | 1 + .../fixtures/import_import_nonexisting.p8 | 1 + .../test/fixtures/import_nonexisting.p8 | 1 + 8 files changed, 208 insertions(+), 138 deletions(-) create mode 100644 compilerAst/test/TestModuleImporter.kt create mode 100644 compilerAst/test/fixtures/file_with_syntax_error.p8 create mode 100644 compilerAst/test/fixtures/import_file_with_syntax_error.p8 create mode 100644 compilerAst/test/fixtures/import_import_nonexisting.p8 create mode 100644 compilerAst/test/fixtures/import_nonexisting.p8 diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index d9a220344..4e1dc4f4c 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -16,7 +16,6 @@ import java.nio.file.Path import java.nio.file.Paths -class ParsingFailedError(override var message: String) : Exception(message) fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') @@ -56,7 +55,7 @@ class ModuleImporter(private val program: Program, } private fun importModule(stream: CharStream, modulePath: Path): Module { - val parser = Prog8Parser() + val parser = Prog8Parser val sourceText = stream.toString() val moduleAst = parser.parseModule(sourceText) moduleAst.program = program diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 5a7a4dfc6..d3d10eea4 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -1,47 +1,50 @@ package prog8.parser import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.misc.ParseCancellationException -import prog8.ast.antlr.toAst import prog8.ast.Module +import prog8.ast.antlr.toAst +import prog8.ast.base.Position +import java.nio.file.Path -class Prog8ErrorStrategy: BailErrorStrategy() { - override fun recover(recognizer: Parser?, e: RecognitionException?) { - try { - // let it - super.recover(recognizer, e) // fills in exception e in all the contexts - // ...then throws ParseCancellationException, which is - // *deliberately* not a RecognitionException. However, we don't try any - // error recovery, therefore report an error in this case, too. - } catch (pce: ParseCancellationException) { - reportError(recognizer, e) - } - } +open class ParsingFailedError(override var message: String) : Exception(message) - override fun recoverInline(recognizer: Parser?): Token { - throw InputMismatchException(recognizer) +class ParseError(override var message: String, val position: Position, cause: RuntimeException) + : ParsingFailedError("${position.toClickableStr()}$message") { + init { + initCause(cause) } } -object ThrowErrorListener: BaseErrorListener() { - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - throw ParsingFailedError("$e: $msg") - } +private fun RecognitionException.getPosition(provenance: String) : Position { + val offending = this.offendingToken + val line = offending.line + val beginCol = offending.charPositionInLine + val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? + val pos = Position(provenance, line, beginCol, endCol) + return pos } -class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorListener) { +object Prog8Parser { - fun parseModule(sourceText: String): Module { - val chars = CharStreams.fromString(sourceText) + fun parseModule(srcPath: Path): Module { + return parseModule(CharStreams.fromPath(srcPath), srcPath.fileName.toString()) + } + + fun parseModule(srcText: String): Module { + return parseModule(CharStreams.fromString(srcText), "") + } + + private fun parseModule(chars: CharStream, provenance: String): Module { + val antlrErrorListener = AntlrErrorListener(provenance) val lexer = Prog8ANTLRLexer(chars) lexer.removeErrorListeners() - lexer.addErrorListener(errorListener) + lexer.addErrorListener(antlrErrorListener) val tokens = CommonTokenStream(lexer) val parser = Prog8ANTLRParser(tokens) - parser.errorHandler = Prog8ErrorStrategy() + parser.errorHandler = Prog8ErrorStrategy parser.removeErrorListeners() - parser.addErrorListener(errorListener) + parser.addErrorListener(antlrErrorListener) val parseTree = parser.module() val moduleName = "anonymous" @@ -54,4 +57,42 @@ class Prog8Parser(private val errorListener: ANTLRErrorListener = ThrowErrorList } return module } + + private object Prog8ErrorStrategy: BailErrorStrategy() { + private fun fillIn(e: RecognitionException?, ctx: ParserRuleContext?) { + var context = ctx + while (context != null) { + context.exception = e + context = context.getParent() + } + } + + override fun reportInputMismatch(recognizer: Parser?, e: InputMismatchException?) { + super.reportInputMismatch(recognizer, e) + } + + override fun recover(recognizer: Parser?, e: RecognitionException?) { + fillIn(e, recognizer!!.context) + reportError(recognizer, e) + } + + override fun recoverInline(recognizer: Parser?): Token { + val e = InputMismatchException(recognizer) + fillIn(e, recognizer!!.context) + reportError(recognizer, e) + throw e + } + } + + private class AntlrErrorListener(val sourceCodeProvenance: String): BaseErrorListener() { + override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { + if (e == null) { + TODO("no RecognitionException - create your own ParseError") + //throw ParseError() + } else { + throw ParseError(msg, e.getPosition(sourceCodeProvenance), e) + } + } + } + } diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestAntlrParser.kt index 9034685a4..d30e15bdd 100644 --- a/compilerAst/test/TestAntlrParser.kt +++ b/compilerAst/test/TestAntlrParser.kt @@ -1,107 +1,30 @@ package prog8tests -import org.antlr.v4.runtime.* -import org.antlr.v4.runtime.misc.ParseCancellationException import org.junit.jupiter.api.Test -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding -import prog8.ast.Program -import prog8.ast.antlr.toAst -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import prog8.ast.statements.Block -import prog8.parser.* +import prog8.parser.ParseError +import prog8.parser.Prog8Parser.parseModule import java.nio.file.Path import kotlin.test.* -class TestAntlrParser { - - class MyErrorListener: ConsoleErrorListener() { - override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { - throw ParsingFailedError("line $line:$charPositionInLine $msg") - } - } - - class MyErrorStrategy: BailErrorStrategy() { - override fun recover(recognizer: Parser?, e: RecognitionException?) { - try { - // let it - super.recover(recognizer, e) // fills in exception e in all the contexts - // ...then throws ParseCancellationException, which is - // *deliberately* not a RecognitionException. However, we don't try any - // error recovery, therefore report an error in this case, too. - } catch (pce: ParseCancellationException) { - reportError(recognizer, e) - } - } - - override fun recoverInline(recognizer: Parser?): Token { - throw InputMismatchException(recognizer) - } - } - - private fun parseModule(srcText: String): Prog8ANTLRParser.ModuleContext { - return parseModule(CharStreams.fromString(srcText)) - } - - private fun parseModule(srcFile: Path): Prog8ANTLRParser.ModuleContext { - return parseModule(CharStreams.fromPath(srcFile)) - } - - private fun parseModule(srcStream: CharStream): Prog8ANTLRParser.ModuleContext { - val errorListener = MyErrorListener() - val lexer = Prog8ANTLRLexer(srcStream) - lexer.removeErrorListeners() - lexer.addErrorListener(errorListener) - val tokens = CommonTokenStream(lexer) - val parser = Prog8ANTLRParser(tokens) - parser.errorHandler = MyErrorStrategy() - parser.removeErrorListeners() - parser.addErrorListener(errorListener) - return parser.module() - } - - object DummyEncoding: IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("Not yet implemented") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("Not yet implemented") - } - } - - object DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - object DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } +class TestProg8Parser { @Test fun testModuleSourceNeedNotEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) - // before the fix, Prog8ANTLRParser would have reported (thrown) "missing at ''" - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + // #45: Prog8ANTLRParser would report (throw) "missing at ''" + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test fun testModuleSourceMayEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -114,9 +37,9 @@ class TestAntlrParser { // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" - assertFailsWith { parseModule(srcBad) } - val parseTree = parseModule(srcGood) - assertEquals(parseTree.block().size, 2) + assertFailsWith { parseModule(srcBad) } + val module = parseModule(srcGood) + assertEquals(2, module.statements.size) } @Test @@ -143,8 +66,8 @@ class TestAntlrParser { "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 2) + val module = parseModule(srcText) + assertEquals(2, module.statements.size) } @Test @@ -158,8 +81,8 @@ class TestAntlrParser { blockA { } """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -175,8 +98,8 @@ class TestAntlrParser { blockB { } """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 2) + val module = parseModule(srcText) + assertEquals(2, module.statements.size) } @Test @@ -190,8 +113,8 @@ class TestAntlrParser { ; comment """ - val parseTree = parseModule(srcText) - assertEquals(parseTree.block().size, 1) + val module = parseModule(srcText) + assertEquals(1, module.statements.size) } @Test @@ -199,14 +122,14 @@ class TestAntlrParser { // issue: #47 // block and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" blockA { } blockB { } """) } // block and directive - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" blockB { } %import textio """) } @@ -215,37 +138,58 @@ class TestAntlrParser { // Leaving them in anyways. // dir and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" %import textio blockB { } """) } - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(""" %import textio %import syslib """) } } - /* @Test - fun testImportLibraryModule() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + fun testErrorLocationForSourceFromString() { + val srcText = "bad * { }\n" - //assertFailsWith(){ importer.importLibraryModule("import_file_with_syntax_error") } + assertFailsWith { parseModule(srcText) } + try { + parseModule(srcText) + } catch (e: ParseError) { + // Note: assertContains expects *actual* value first + assertContains(e.position.file, Regex("^$")) + assertEquals(1, e.position.line, "line; should be 1-based") + assertEquals(4, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(4, e.position.endCol, "endCol; should be 0-based") + } + } + + @Test + fun testErrorLocationForSourceFromPath() { + val filename = "file_with_syntax_error.p8" + val path = Path.of("test", "fixtures", filename) + + assertFailsWith { parseModule(path) } + try { + parseModule(path) + } catch (e: ParseError) { + assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } } - */ @Test fun testProg8Ast() { - val parseTree = parseModule(""" + val module = parseModule(""" main { sub start() { return } } """) - val ast = parseTree.toAst("test", Path.of(""), DummyEncoding) - assertIs(ast.statements.first()) - assertEquals((ast.statements.first() as Block).name, "main") + assertIs(module.statements.first()) + assertEquals((module.statements.first() as Block).name, "main") } } diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt new file mode 100644 index 000000000..3b18d5cae --- /dev/null +++ b/compilerAst/test/TestModuleImporter.kt @@ -0,0 +1,81 @@ +package prog8tests + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.IStringEncoding +import prog8.ast.Program +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import prog8.parser.ModuleImporter +import prog8.parser.ParseError +import java.nio.file.Path +import org.junit.jupiter.api.Test +import kotlin.test.* + + +class TestModuleImporter { + + object DummyEncoding: IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("Not yet implemented") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("Not yet implemented") + } + } + + object DummyFunctions: IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() + } + + object DummyMemsizer: IMemSizer { + override fun memorySize(dt: DataType): Int = 0 + } + + @Test + fun testImportModuleWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val filename = "file_with_syntax_error.p8" + val act = { importer.importModule(Path.of("test", "fixtures", filename )) } + + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + + @Test + fun testImportLibraryModuleImportingBadModule() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val importing = "import_file_with_syntax_error" + val imported = "file_with_syntax_error" + val act = { importer.importLibraryModule(importing) } + + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals(imported + ".p8", e.position.file, "provenance; should be the importED file's name, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + +} diff --git a/compilerAst/test/fixtures/file_with_syntax_error.p8 b/compilerAst/test/fixtures/file_with_syntax_error.p8 new file mode 100644 index 000000000..6b9930fe0 --- /dev/null +++ b/compilerAst/test/fixtures/file_with_syntax_error.p8 @@ -0,0 +1,2 @@ +; test expects the following 2nd (!) line: +bad { } ; -> missing EOL at '}' (ie: *after* the '{') diff --git a/compilerAst/test/fixtures/import_file_with_syntax_error.p8 b/compilerAst/test/fixtures/import_file_with_syntax_error.p8 new file mode 100644 index 000000000..a6de5368f --- /dev/null +++ b/compilerAst/test/fixtures/import_file_with_syntax_error.p8 @@ -0,0 +1 @@ +%import file_with_syntax_error diff --git a/compilerAst/test/fixtures/import_import_nonexisting.p8 b/compilerAst/test/fixtures/import_import_nonexisting.p8 new file mode 100644 index 000000000..c9c418fa3 --- /dev/null +++ b/compilerAst/test/fixtures/import_import_nonexisting.p8 @@ -0,0 +1 @@ +%import import_nonexisting diff --git a/compilerAst/test/fixtures/import_nonexisting.p8 b/compilerAst/test/fixtures/import_nonexisting.p8 new file mode 100644 index 000000000..3d028ba09 --- /dev/null +++ b/compilerAst/test/fixtures/import_nonexisting.p8 @@ -0,0 +1 @@ +%import i_do_not_exist From ce554f771820539b63ee2c6efa84eb6c4304bf37 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 3 Jul 2021 17:17:31 +0200 Subject: [PATCH 04/68] * rename test file --- compilerAst/test/{TestAntlrParser.kt => TestProg8Parser.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename compilerAst/test/{TestAntlrParser.kt => TestProg8Parser.kt} (100%) diff --git a/compilerAst/test/TestAntlrParser.kt b/compilerAst/test/TestProg8Parser.kt similarity index 100% rename from compilerAst/test/TestAntlrParser.kt rename to compilerAst/test/TestProg8Parser.kt From b071a58ca775b194ccfe709e8f315b8b1b37098f Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 20:27:04 +0200 Subject: [PATCH 05/68] + add tests - 4 failing in TestModuleImporter --- compilerAst/test/TestModuleImporter.kt | 60 ++++++++++++++++++++++++-- compilerAst/test/TestProg8Parser.kt | 42 ++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 3b18d5cae..04e296d62 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -13,9 +13,12 @@ import prog8.parser.ModuleImporter import prog8.parser.ParseError import java.nio.file.Path import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import kotlin.io.path.isRegularFile import kotlin.test.* +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestModuleImporter { object DummyEncoding: IStringEncoding { @@ -39,6 +42,7 @@ class TestModuleImporter { override fun memorySize(dt: DataType): Int = 0 } + @Test fun testImportModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) @@ -59,7 +63,53 @@ class TestModuleImporter { } @Test - fun testImportLibraryModuleImportingBadModule() { + fun testImportModuleWithImportingModuleWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val importing = Path.of("test", "fixtures", "import_file_with_syntax_error.p8") + val imported = Path.of("test", "fixtures", "file_with_syntax_error.p8") + + val act = { importer.importModule(importing) } + + assertTrue(importing.isRegularFile(), "sanity check: should be regular file") + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals(imported.fileName.toString(), e.position.file, "provenance; should be the importED file's filename, incl. extension '.p8'") + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + + + @Test + fun testImportLibraryModuleWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val filename = "file_with_syntax_error" + val act = { importer.importLibraryModule(filename) } + + assertFailsWith { act() } + try { + act() + } catch (e: ParseError) { + assertEquals( + filename + ".p8", + e.position.file, + "provenance; should be the path's filename, incl. extension '.p8'" + ) + assertEquals(2, e.position.line, "line; should be 1-based") + assertEquals(6, e.position.startCol, "startCol; should be 0-based") + assertEquals(6, e.position.endCol, "endCol; should be 0-based") + } + } + + @Test + fun testImportLibraryModuleWithImportingBadModule() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) @@ -71,9 +121,13 @@ class TestModuleImporter { try { act() } catch (e: ParseError) { - assertEquals(imported + ".p8", e.position.file, "provenance; should be the importED file's name, incl. extension '.p8'") + assertEquals( + imported + ".p8", + e.position.file, + "provenance; should be the importED file's name, incl. extension '.p8'" + ) assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) + assertEquals(6, e.position.startCol, "startCol; should be 0-based") assertEquals(6, e.position.endCol, "endCol; should be 0-based") } } diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index d30e15bdd..749c1c7d1 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -3,8 +3,13 @@ package prog8tests import org.junit.jupiter.api.Test import prog8.ast.statements.Block import prog8.parser.ParseError +import prog8.parser.Prog8Parser import prog8.parser.Prog8Parser.parseModule import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isReadable +import kotlin.io.path.isRegularFile import kotlin.test.* class TestProg8Parser { @@ -148,6 +153,43 @@ class TestProg8Parser { """) } } + @Test + fun testParseModuleWithDirectoryPath() { + val srcPath = Path.of("test", "fixtures") + assertTrue(srcPath.isDirectory(), "sanity check: should be a directory") + assertFailsWith { Prog8Parser.parseModule(srcPath) } + } + + @Test + fun testParseModuleWithNonExistingPath() { + val srcPath = Path.of("test", "fixtures", "i_do_not_exist") + assertFalse(srcPath.exists(), "sanity check: file should not exist") + assertFailsWith { Prog8Parser.parseModule(srcPath) } + } + + @Test + fun testParseModuleWithPathMissingExtension_p8() { + val srcPathWithoutExt = Path.of("test", "fixtures", "file_with_syntax_error") + val srcPathWithExt = Path.of(srcPathWithoutExt.toString() + ".p8") + assertTrue(srcPathWithExt.isRegularFile(), "sanity check: should be normal file") + assertTrue(srcPathWithExt.isReadable(), "sanity check: should be readable") + assertFailsWith { Prog8Parser.parseModule(srcPathWithoutExt) } + } + + @Test + fun testParseModuleWithStringShouldNotLookAtImports() { + val srcText = "%import i_do_not_exist" + val module = Prog8Parser.parseModule(srcText) + assertEquals(1, module.statements.size) + } + + @Test + fun testParseModuleWithPathShouldNotLookAtImports() { + val srcPath = Path.of("test", "fixtures", "import_nonexisting.p8") + val module = Prog8Parser.parseModule(srcPath) + assertEquals(1, module.statements.size) + } + @Test fun testErrorLocationForSourceFromString() { val srcText = "bad * { }\n" From b6f780d70d32d7fd263827ba657752b575b291fb Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 20:30:42 +0200 Subject: [PATCH 06/68] * ModuleImporter: make tests pass --- compilerAst/src/prog8/parser/ModuleParsing.kt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 4e1dc4f4c..fd0bf5f4d 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -4,7 +4,6 @@ import org.antlr.v4.runtime.* import prog8.ast.IStringEncoding import prog8.ast.Module import prog8.ast.Program -import prog8.ast.antlr.toAst import prog8.ast.base.Position import prog8.ast.base.SyntaxError import prog8.ast.statements.Directive @@ -41,10 +40,21 @@ class ModuleImporter(private val program: Program, if(!Files.isReadable(filePath)) throw ParsingFailedError("No such file: $filePath") - val content = filePath.toFile().readText() - val cs = CharStreams.fromString(content) + val module = Prog8Parser.parseModule(filePath) - return importModule(cs, filePath) + module.program = program + module.linkParents(program.namespace) + program.modules.add(module) + + // accept additional imports + val lines = module.statements.toMutableList() + lines.asSequence() + .mapIndexed { i, it -> i to it } + .filter { (it.second as? Directive)?.directive == "%import" } + .forEach { executeImportDirective(it.second as Directive, filePath) } + + module.statements = lines + return module } fun importLibraryModule(name: String): Module? { From cd4ed8765b66d7fcd9a43d116ff1b294daac37f3 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 20:45:37 +0200 Subject: [PATCH 07/68] + add tests for importModule(Path) with invalid path (non-existent or directory) - *failing* --- compilerAst/test/TestModuleImporter.kt | 39 +++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 04e296d62..a1674dfe3 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -14,7 +14,7 @@ import prog8.parser.ParseError import java.nio.file.Path import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance -import kotlin.io.path.isRegularFile +import kotlin.io.path.* import kotlin.test.* @@ -43,6 +43,43 @@ class TestModuleImporter { } + @Test + fun testImportModuleWithNonExistingPath() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val srcPath = Path.of("test", "fixtures", "i_do_not_exist") + + assertFalse(srcPath.exists(), "sanity check: file should not exist") + assertFailsWith { importer.importModule(srcPath) } + } + + @Test + fun testImportModuleWithDirectoryPath() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val srcPath = Path.of("test", "fixtures") + + assertTrue(srcPath.isDirectory(), "sanity check: should be a directory") + + // fn importModule(Path) used to check *.isReadable()*, but NOT .isRegularFile(): + assertTrue(srcPath.isReadable(), "sanity check: should still be readable") + + assertFailsWith { importer.importModule(srcPath) } + } + + @Test + fun testImportLibraryModuleWithNonExistingPath() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + + val srcPath = Path.of("i_do_not_exist.p8") + + assertFalse(srcPath.exists(), "sanity check: file should not exist") + assertFailsWith { importer.importLibraryModule(srcPath.nameWithoutExtension) } + } + @Test fun testImportModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) From d31a88206c266a535f92c856c204c062c62873d2 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 19 Jun 2021 20:48:52 +0200 Subject: [PATCH 08/68] * importModule(Path): make tests pass (TODO: importLibraryModule with non-existent path) --- compilerAst/src/prog8/parser/ModuleParsing.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index fd0bf5f4d..1689fc03e 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -37,8 +37,6 @@ class ModuleImporter(private val program: Program, } else println("") - if(!Files.isReadable(filePath)) - throw ParsingFailedError("No such file: $filePath") val module = Prog8Parser.parseModule(filePath) From 7b89228fa79ca9ad91986e4bffd5d3e82d7ff9f5 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 11:52:09 +0200 Subject: [PATCH 09/68] + add TODOs re ICompilationTarget --- compiler/src/prog8/compiler/target/ICompilationTarget.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/prog8/compiler/target/ICompilationTarget.kt b/compiler/src/prog8/compiler/target/ICompilationTarget.kt index b778e3182..6f71ccd3c 100644 --- a/compiler/src/prog8/compiler/target/ICompilationTarget.kt +++ b/compiler/src/prog8/compiler/target/ICompilationTarget.kt @@ -24,6 +24,8 @@ interface ICompilationTarget: IStringEncoding, IMemSizer { override fun encodeString(str: String, altEncoding: Boolean): List override fun decodeString(bytes: List, altEncoding: Boolean): String + // TODO: rename param target, and also AST node AssignTarget - *different meaning of "target"!* + // TODO: remove param program - can be obtained from AST node fun isInRegularRAM(target: AssignTarget, program: Program): Boolean { val memAddr = target.memoryAddress val arrayIdx = target.arrayindexed From af209ad50ef76bda655f54e46ed1684c4bf09885 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 12:02:36 +0200 Subject: [PATCH 10/68] + intro SourceCode, tying together source code text with its *origin*; Prog8Parser now only accepts this --- compilerAst/src/prog8/parser/ModuleParsing.kt | 46 ++++--- compilerAst/src/prog8/parser/Prog8Parser.kt | 14 +-- compilerAst/src/prog8/parser/SourceCode.kt | 118 ++++++++++++++++++ compilerAst/test/TestModuleImporter.kt | 69 +++++----- compilerAst/test/TestProg8Parser.kt | 109 +++++++--------- compilerAst/test/TestSourceCode.kt | 82 ++++++++++++ compilerAst/test/fixtures/simple_main.p8 | 4 + 7 files changed, 307 insertions(+), 135 deletions(-) create mode 100644 compilerAst/src/prog8/parser/SourceCode.kt create mode 100644 compilerAst/test/TestSourceCode.kt create mode 100644 compilerAst/test/fixtures/simple_main.p8 diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 1689fc03e..69dd07820 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -8,12 +8,12 @@ import prog8.ast.base.Position import prog8.ast.base.SyntaxError import prog8.ast.statements.Directive import prog8.ast.statements.DirectiveArg -import java.io.InputStream +import kotlin.io.FileSystemException +import java.net.URL import java.nio.file.FileSystems import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - +import java.nio.file.Path // TODO: use kotlin.io.paths.Path instead +import java.nio.file.Paths // TODO: use kotlin.io.paths.Path instead fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') @@ -38,7 +38,7 @@ class ModuleImporter(private val program: Program, else println("") - val module = Prog8Parser.parseModule(filePath) + val module = Prog8Parser.parseModule(SourceCode.fromPath(filePath)) module.program = program module.linkParents(program.namespace) @@ -65,7 +65,7 @@ class ModuleImporter(private val program: Program, private fun importModule(stream: CharStream, modulePath: Path): Module { val parser = Prog8Parser val sourceText = stream.toString() - val moduleAst = parser.parseModule(sourceText) + val moduleAst = parser.parseModule(SourceCode.of(sourceText)) moduleAst.program = program moduleAst.linkParents(program.namespace) program.modules.add(moduleAst) @@ -92,16 +92,13 @@ class ModuleImporter(private val program: Program, if(existing!=null) return null - val rsc = tryGetModuleFromResource("$moduleName.p8", compilationTargetName) + val srcCode = tryGetModuleFromResource("$moduleName.p8", compilationTargetName) val importedModule = - if(rsc!=null) { + if (srcCode != null) { // found in resources // load the module from the embedded resource - val (resource, resourcePath) = rsc - resource.use { - println("importing '$moduleName' (library)") - val content = it.reader().readText().replace("\r\n", "\n") - importModule(CharStreams.fromString(content), Module.pathForResource(resourcePath)) - } + println("importing '$moduleName' (library): ${srcCode.origin}") + val path = Path.of(URL(srcCode.origin).file) + importModule(srcCode.getCharStream(), path) } else { val modulePath = tryGetModuleFromFile(moduleName, source, import.position) importModule(modulePath) @@ -120,17 +117,16 @@ class ModuleImporter(private val program: Program, importedModule.statements.addAll(0, directives) } - private fun tryGetModuleFromResource(name: String, compilationTargetName: String): Pair? { - val targetSpecificPath = "/prog8lib/$compilationTargetName/$name" - val targetSpecificResource = object{}.javaClass.getResourceAsStream(targetSpecificPath) - if(targetSpecificResource!=null) - return Pair(targetSpecificResource, targetSpecificPath) - - val generalPath = "/prog8lib/$name" - val generalResource = object{}.javaClass.getResourceAsStream(generalPath) - if(generalResource!=null) - return Pair(generalResource, generalPath) - + private fun tryGetModuleFromResource(name: String, compilationTargetName: String): SourceCode? { + // try target speficic first + try { + return SourceCode.fromResources("/prog8lib/$compilationTargetName/$name") + } catch (e: FileSystemException) { + } + try { + return SourceCode.fromResources("/prog8lib/$name") + } catch (e: FileSystemException) { + } return null } diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index d3d10eea4..45c684d60 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -27,17 +27,9 @@ private fun RecognitionException.getPosition(provenance: String) : Position { object Prog8Parser { - fun parseModule(srcPath: Path): Module { - return parseModule(CharStreams.fromPath(srcPath), srcPath.fileName.toString()) - } - - fun parseModule(srcText: String): Module { - return parseModule(CharStreams.fromString(srcText), "") - } - - private fun parseModule(chars: CharStream, provenance: String): Module { - val antlrErrorListener = AntlrErrorListener(provenance) - val lexer = Prog8ANTLRLexer(chars) + fun parseModule(src: SourceCode): Module { + val antlrErrorListener = AntlrErrorListener(src.origin) + val lexer = Prog8ANTLRLexer(src.getCharStream()) lexer.removeErrorListeners() lexer.addErrorListener(antlrErrorListener) val tokens = CommonTokenStream(lexer) diff --git a/compilerAst/src/prog8/parser/SourceCode.kt b/compilerAst/src/prog8/parser/SourceCode.kt new file mode 100644 index 000000000..02b3f721b --- /dev/null +++ b/compilerAst/src/prog8/parser/SourceCode.kt @@ -0,0 +1,118 @@ +package prog8.parser + +import org.antlr.v4.runtime.CharStream +import org.antlr.v4.runtime.CharStreams +import java.io.File +import java.nio.file.Path +import kotlin.io.path.* + +/** + * Encapsulates - and ties together - actual source code (=text) + * and its [origin]. + */ +abstract class SourceCode() { + /** + * Where this [SourceCode] instance came from. + * This can be one of the following: + * * a normal string representation of a [java.nio.file.Path], if it originates from a file (see [fromPath]) + * * `` if was created via [of] + * * `` if it came from resources (see [fromResources]) + */ + abstract val origin: String + abstract fun getCharStream(): CharStream + + /** + * The source code as plain string. + * *Note: this is meant for testing and debugging, do NOT use in application code!* + */ + fun asString() = this.getCharStream().toString() + + /** + * Deliberately does NOT return the actual text. + * Use [getCharStream]. + */ + final override fun toString() = super.toString() + + + + + + // "static" factory methods + companion object { + + /** + * Turn a plain String into a [SourceCode] object. + * [origin] will be something like ``. + */ + fun of(text: String): SourceCode { + return object : SourceCode() { + override val origin = "" + override fun getCharStream(): CharStream { + return CharStreams.fromString(text) + } + } + } + + /** + * Get [SourceCode] from the file represented by the specified Path. + * This does not actually *access* the file, but it does check + * whether it + * * exists + * * is a regular file (ie: not a directory) + * * and is actually readable + * + * [origin] will be the given path in absolute and normalized form. + * @throws NoSuchFileException if the file does not exist + * @throws AccessDeniedException if the given path points to a directory or the file is non-readable for some other reason + */ + fun fromPath(path: Path): SourceCode { + if (!path.exists()) + throw NoSuchFileException(path.toFile()) + if (path.isDirectory()) + throw AccessDeniedException(path.toFile(), reason = "Not a file but a directory") + if (!path.isReadable()) + throw AccessDeniedException(path.toFile(), reason = "Is not readable") + val normalized = path.normalize() + return object : SourceCode() { + override val origin = normalized.absolutePathString() + override fun getCharStream(): CharStream { + return CharStreams.fromPath(normalized) + } + } + } + + /** + * [origin]: `` for a given `pathString` of "x/y/z.p8" + */ + fun fromResources(pathString: String): SourceCode { + val path = Path.of(pathString).normalize() + val sep = "/" + val normalized = sep + path.toMutableList().joinToString(sep) + val rscURL = object{}.javaClass.getResource(normalized) + if (rscURL == null) { + val rscRoot = object{}.javaClass.getResource("/") + throw NoSuchFileException( + File(normalized), + reason = "looked in resources rooted at $rscRoot") + } + return object : SourceCode() { + override val origin = "" + override fun getCharStream(): CharStream { + val inpStr = object{}.javaClass.getResourceAsStream(normalized) + val chars = CharStreams.fromStream(inpStr) + return chars + } + } + } + + // TODO: possibly more, like fromURL(..) +/* // For `jar:..` URLs + // see https://stackoverflow.com/questions/22605666/java-access-files-in-jar-causes-java-nio-file-filesystemnotfoundexception + var url = URL("jar:file:/E:/x16/prog8(meisl)/compiler/build/libs/prog8compiler-7.0-BETA3-all.jar!/prog8lib/c64/textio.p8") + val uri = url.toURI() + val parts = uri.toString().split("!") + val fs = FileSystems.newFileSystem(URI.create(parts[0]), mutableMapOf(Pair("", "")) ) + val path = fs.getPath(parts[1]) +*/ + } +} diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index a1674dfe3..6fe9b6a73 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -1,5 +1,11 @@ package prog8tests +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import kotlin.test.* +import java.nio.file.Path // TODO: use kotlin.io.path.Path instead +import kotlin.io.path.* + import prog8.ast.IBuiltinFunctions import prog8.ast.IMemSizer import prog8.ast.IStringEncoding @@ -11,11 +17,6 @@ import prog8.ast.expressions.InferredTypes import prog8.ast.expressions.NumericLiteralValue import prog8.parser.ModuleImporter import prog8.parser.ParseError -import java.nio.file.Path -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import kotlin.io.path.* -import kotlin.test.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -51,7 +52,7 @@ class TestModuleImporter { val srcPath = Path.of("test", "fixtures", "i_do_not_exist") assertFalse(srcPath.exists(), "sanity check: file should not exist") - assertFailsWith { importer.importModule(srcPath) } + assertFailsWith { importer.importModule(srcPath) } } @Test @@ -66,18 +67,7 @@ class TestModuleImporter { // fn importModule(Path) used to check *.isReadable()*, but NOT .isRegularFile(): assertTrue(srcPath.isReadable(), "sanity check: should still be readable") - assertFailsWith { importer.importModule(srcPath) } - } - - @Test - fun testImportLibraryModuleWithNonExistingPath() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) - - val srcPath = Path.of("i_do_not_exist.p8") - - assertFalse(srcPath.exists(), "sanity check: file should not exist") - assertFailsWith { importer.importLibraryModule(srcPath.nameWithoutExtension) } + assertFailsWith { importer.importModule(srcPath) } } @Test @@ -86,13 +76,14 @@ class TestModuleImporter { val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) val filename = "file_with_syntax_error.p8" - val act = { importer.importModule(Path.of("test", "fixtures", filename )) } + val path = Path.of("test", "fixtures", filename) + val act = { importer.importModule(path) } assertFailsWith { act() } try { act() } catch (e: ParseError) { - assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(path.absolutePathString(), e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) assertEquals(6, e.position.endCol, "endCol; should be 0-based") @@ -114,13 +105,28 @@ class TestModuleImporter { try { act() } catch (e: ParseError) { - assertEquals(imported.fileName.toString(), e.position.file, "provenance; should be the importED file's filename, incl. extension '.p8'") + val expectedProvenance = imported.absolutePathString() + assertEquals(expectedProvenance, e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) assertEquals(6, e.position.endCol, "endCol; should be 0-based") } } + @Test + fun testImportLibraryModuleWithNonExistingName() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val filenameNoExt = "i_do_not_exist" + val filenameWithExt = filenameNoExt + ".p8" + val srcPathNoExt = Path.of("test", "fixtures", filenameNoExt) + val srcPathWithExt = Path.of("test", "fixtures", filenameWithExt) + + assertFalse(srcPathNoExt.exists(), "sanity check: file should not exist") + assertFalse(srcPathWithExt.exists(), "sanity check: file should not exist") + assertFailsWith { importer.importLibraryModule(filenameNoExt) } + assertFailsWith { importer.importLibraryModule(filenameWithExt) } + } @Test fun testImportLibraryModuleWithSyntaxError() { @@ -128,17 +134,16 @@ class TestModuleImporter { val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) val filename = "file_with_syntax_error" + val act = { importer.importLibraryModule(filename) } assertFailsWith { act() } try { act() } catch (e: ParseError) { - assertEquals( - filename + ".p8", - e.position.file, - "provenance; should be the path's filename, incl. extension '.p8'" - ) + val expectedProvenance = Path.of("test", "fixtures", filename + ".p8") + .absolutePathString() + assertEquals(expectedProvenance, e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based") assertEquals(6, e.position.endCol, "endCol; should be 0-based") @@ -148,7 +153,8 @@ class TestModuleImporter { @Test fun testImportLibraryModuleWithImportingBadModule() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val libdirs = listOf("./test/fixtures") + val importer = ModuleImporter(program, DummyEncoding, "blah", libdirs) val importing = "import_file_with_syntax_error" val imported = "file_with_syntax_error" @@ -158,11 +164,10 @@ class TestModuleImporter { try { act() } catch (e: ParseError) { - assertEquals( - imported + ".p8", - e.position.file, - "provenance; should be the importED file's name, incl. extension '.p8'" - ) + val expectedProvenance = Path.of(libdirs[0], imported + ".p8") + .normalize() + .absolutePathString() + assertEquals(expectedProvenance, e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based") assertEquals(6, e.position.endCol, "endCol; should be 0-based") diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 749c1c7d1..381feffa6 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,26 +1,24 @@ package prog8tests import org.junit.jupiter.api.Test +import kotlin.test.* +import java.nio.file.Path // TODO: use kotlin.io.path.Path instead +import kotlin.io.path.* import prog8.ast.statements.Block import prog8.parser.ParseError -import prog8.parser.Prog8Parser import prog8.parser.Prog8Parser.parseModule -import java.nio.file.Path -import kotlin.io.path.exists -import kotlin.io.path.isDirectory -import kotlin.io.path.isReadable -import kotlin.io.path.isRegularFile -import kotlin.test.* +import prog8.parser.SourceCode + class TestProg8Parser { @Test fun testModuleSourceNeedNotEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) - val srcText = "foo {" + nl + "}" // source ends with '}' (= NO newline, issue #40) + val src = SourceCode.of("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) // #45: Prog8ANTLRParser would report (throw) "missing at ''" - val module = parseModule(srcText) + val module = parseModule(src) assertEquals(1, module.statements.size) } @@ -28,7 +26,7 @@ class TestProg8Parser { fun testModuleSourceMayEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) - val module = parseModule(srcText) + val module = parseModule(SourceCode.of(srcText)) assertEquals(1, module.statements.size) } @@ -42,8 +40,8 @@ class TestProg8Parser { // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" - assertFailsWith { parseModule(srcBad) } - val module = parseModule(srcGood) + assertFailsWith { parseModule(SourceCode.of(srcBad)) } + val module = parseModule(SourceCode.of(srcGood)) assertEquals(2, module.statements.size) } @@ -71,7 +69,7 @@ class TestProg8Parser { "}" + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) - val module = parseModule(srcText) + val module = parseModule(SourceCode.of(srcText)) assertEquals(2, module.statements.size) } @@ -86,7 +84,7 @@ class TestProg8Parser { blockA { } """ - val module = parseModule(srcText) + val module = parseModule(SourceCode.of(srcText)) assertEquals(1, module.statements.size) } @@ -103,7 +101,7 @@ class TestProg8Parser { blockB { } """ - val module = parseModule(srcText) + val module = parseModule(SourceCode.of(srcText)) assertEquals(2, module.statements.size) } @@ -118,7 +116,7 @@ class TestProg8Parser { ; comment """ - val module = parseModule(srcText) + val module = parseModule(SourceCode.of(srcText)) assertEquals(1, module.statements.size) } @@ -127,66 +125,43 @@ class TestProg8Parser { // issue: #47 // block and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(SourceCode.of(""" blockA { } blockB { } - """) } + """)) } // block and directive - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(SourceCode.of(""" blockB { } %import textio - """) } + """)) } // The following two are bogus due to directive *args* expected to follow the directive name. // Leaving them in anyways. // dir and block - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(SourceCode.of(""" %import textio blockB { } - """) } + """)) } - assertFailsWith{ parseModule(""" + assertFailsWith{ parseModule(SourceCode.of(""" %import textio %import syslib - """) } + """)) } } @Test - fun testParseModuleWithDirectoryPath() { - val srcPath = Path.of("test", "fixtures") - assertTrue(srcPath.isDirectory(), "sanity check: should be a directory") - assertFailsWith { Prog8Parser.parseModule(srcPath) } - } + fun parseModuleShouldNotLookAtImports() { + val imported = "i_do_not_exist" + val pathNoExt = Path.of(imported).absolute() + val pathWithExt = Path.of("${pathNoExt}.p8") + val text = "%import $imported" - @Test - fun testParseModuleWithNonExistingPath() { - val srcPath = Path.of("test", "fixtures", "i_do_not_exist") - assertFalse(srcPath.exists(), "sanity check: file should not exist") - assertFailsWith { Prog8Parser.parseModule(srcPath) } - } + assertFalse(pathNoExt.exists(), "sanity check: file should not exist: $pathNoExt") + assertFalse(pathWithExt.exists(), "sanity check: file should not exist: $pathWithExt") - @Test - fun testParseModuleWithPathMissingExtension_p8() { - val srcPathWithoutExt = Path.of("test", "fixtures", "file_with_syntax_error") - val srcPathWithExt = Path.of(srcPathWithoutExt.toString() + ".p8") - assertTrue(srcPathWithExt.isRegularFile(), "sanity check: should be normal file") - assertTrue(srcPathWithExt.isReadable(), "sanity check: should be readable") - assertFailsWith { Prog8Parser.parseModule(srcPathWithoutExt) } - } - - @Test - fun testParseModuleWithStringShouldNotLookAtImports() { - val srcText = "%import i_do_not_exist" - val module = Prog8Parser.parseModule(srcText) - assertEquals(1, module.statements.size) - } - - @Test - fun testParseModuleWithPathShouldNotLookAtImports() { - val srcPath = Path.of("test", "fixtures", "import_nonexisting.p8") - val module = Prog8Parser.parseModule(srcPath) + val module = parseModule(SourceCode.of(text)) assertEquals(1, module.statements.size) } @@ -194,9 +169,9 @@ class TestProg8Parser { fun testErrorLocationForSourceFromString() { val srcText = "bad * { }\n" - assertFailsWith { parseModule(srcText) } + assertFailsWith { parseModule(SourceCode.of(srcText)) } try { - parseModule(srcText) + parseModule(SourceCode.of(srcText)) } catch (e: ParseError) { // Note: assertContains expects *actual* value first assertContains(e.position.file, Regex("^$")) @@ -211,11 +186,11 @@ class TestProg8Parser { val filename = "file_with_syntax_error.p8" val path = Path.of("test", "fixtures", filename) - assertFailsWith { parseModule(path) } + assertFailsWith { parseModule(SourceCode.fromPath(path)) } try { - parseModule(path) + parseModule(SourceCode.fromPath(path)) } catch (e: ParseError) { - assertEquals(filename, e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") + assertEquals(path.absolutePathString(), e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) assertEquals(6, e.position.endCol, "endCol; should be 0-based") @@ -224,13 +199,13 @@ class TestProg8Parser { @Test fun testProg8Ast() { - val module = parseModule(""" -main { - sub start() { - return - } -} -""") + val module = parseModule(SourceCode.of(""" + main { + sub start() { + return + } + } + """)) assertIs(module.statements.first()) assertEquals((module.statements.first() as Block).name, "main") } diff --git a/compilerAst/test/TestSourceCode.kt b/compilerAst/test/TestSourceCode.kt new file mode 100644 index 000000000..7a20767f2 --- /dev/null +++ b/compilerAst/test/TestSourceCode.kt @@ -0,0 +1,82 @@ +package prog8tests + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import kotlin.test.* +import java.nio.file.Path // TODO: use kotlin.io.path.Path instead +import kotlin.io.path.* + +import prog8.parser.SourceCode + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestSourceCode { + + @Test + fun testFactoryMethod_Of() { + val text = """ + main { } + """.trimIndent() + val src = SourceCode.of(text) + val actualText = src.getCharStream().toString() + + assertContains(src.origin, Regex("^$")) + assertEquals(text, actualText) + } + + @Test + fun testFromPathWithNonExistingPath() { + val filename = "i_do_not_exist.p8" + val path = Path.of("test", "fixtures", filename) + + assertFalse(path.exists(), "sanity check: file should not exist: ${path.absolute()}") + assertFailsWith { SourceCode.fromPath(path) } + } + + @Test + fun testFromPathWithMissingExtension_p8() { + val pathWithoutExt = Path.of("test", "fixtures", "simple_main") + val pathWithExt = Path.of(pathWithoutExt.toString() + ".p8") + + assertTrue(pathWithExt.isRegularFile(), "sanity check: should be normal file: ${pathWithExt.absolute()}") + assertTrue(pathWithExt.isReadable(), "sanity check: should be readable: ${pathWithExt.absolute()}") + assertFailsWith { SourceCode.fromPath(pathWithoutExt) } + } + + @Test + fun testFromPathWithDirectory() { + val path = Path.of("test", "fixtures") + + assertTrue(path.isDirectory(), "sanity check: should be a directory") + assertFailsWith { SourceCode.fromPath(path) } + } + + @Test + fun testFromPathWithExistingPath() { + val filename = "simple_main.p8" + val path = Path.of("test", "fixtures", filename) + val src = SourceCode.fromPath(path) + + val expectedOrigin = path.normalize().absolutePathString() + assertEquals(expectedOrigin, src.origin) + + val expectedSrcText = path.toFile().readText() + val actualSrcText = src.getCharStream().toString() + assertEquals(expectedSrcText, actualSrcText) + } + + @Test + fun testFromPathWithExistingNonNormalizedPath() { + val filename = "simple_main.p8" + val path = Path.of(".", "test", "..", "test", "fixtures", filename) + val src = SourceCode.fromPath(path) + + val expectedOrigin = path.normalize().absolutePathString() + assertEquals(expectedOrigin, src.origin) + + val expectedSrcText = path.toFile().readText() + val actualSrcText = src.getCharStream().toString() + assertEquals(expectedSrcText, actualSrcText) + } + +} diff --git a/compilerAst/test/fixtures/simple_main.p8 b/compilerAst/test/fixtures/simple_main.p8 new file mode 100644 index 000000000..afaa79f93 --- /dev/null +++ b/compilerAst/test/fixtures/simple_main.p8 @@ -0,0 +1,4 @@ +main { + sub start() { + } +} From fa5ecd6495222272f9d85064c8fb8e5550de87a2 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 13:16:32 +0200 Subject: [PATCH 11/68] * refactor ModuleImporter: throw the proper NoSuchFileException if import isn't found, return SourceCode? from both, tryGetModuleFromResource and tryGetModuleFromFile --- compilerAst/src/prog8/parser/ModuleParsing.kt | 67 ++++++++++--------- compilerAst/src/prog8/parser/Prog8Parser.kt | 4 +- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 69dd07820..d54bc6925 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -1,6 +1,5 @@ package prog8.parser -import org.antlr.v4.runtime.* import prog8.ast.IStringEncoding import prog8.ast.Module import prog8.ast.Program @@ -8,18 +7,14 @@ import prog8.ast.base.Position import prog8.ast.base.SyntaxError import prog8.ast.statements.Directive import prog8.ast.statements.DirectiveArg +import java.io.File import kotlin.io.FileSystemException -import java.net.URL -import java.nio.file.FileSystems -import java.nio.file.Files import java.nio.file.Path // TODO: use kotlin.io.paths.Path instead import java.nio.file.Paths // TODO: use kotlin.io.paths.Path instead fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') -internal fun pathFrom(stringPath: String, vararg rest: String): Path = FileSystems.getDefault().getPath(stringPath, *rest) - class ModuleImporter(private val program: Program, private val encoder: IStringEncoding, @@ -28,7 +23,7 @@ class ModuleImporter(private val program: Program, fun importModule(filePath: Path): Module { print("importing '${moduleName(filePath.fileName)}'") - if(filePath.parent!=null) { + if (filePath.parent != null) { // TODO: use Path.relativize var importloc = filePath.toString() val curdir = Paths.get("").toAbsolutePath().toString() if(importloc.startsWith(curdir)) @@ -49,7 +44,7 @@ class ModuleImporter(private val program: Program, lines.asSequence() .mapIndexed { i, it -> i to it } .filter { (it.second as? Directive)?.directive == "%import" } - .forEach { executeImportDirective(it.second as Directive, filePath) } + .forEach { executeImportDirective(it.second as Directive, module) } module.statements = lines return module @@ -59,13 +54,12 @@ class ModuleImporter(private val program: Program, val import = Directive("%import", listOf( DirectiveArg("", name, 42, position = Position("<<>>", 0, 0, 0)) ), Position("<<>>", 0, 0, 0)) - return executeImportDirective(import, Paths.get("")) + return executeImportDirective(import, null) } - private fun importModule(stream: CharStream, modulePath: Path): Module { - val parser = Prog8Parser - val sourceText = stream.toString() - val moduleAst = parser.parseModule(SourceCode.of(sourceText)) + //private fun importModule(stream: CharStream, modulePath: Path, isLibrary: Boolean): Module { + private fun importModule(src: SourceCode) : Module { + val moduleAst = Prog8Parser.parseModule(src) moduleAst.program = program moduleAst.linkParents(program.namespace) program.modules.add(moduleAst) @@ -75,13 +69,13 @@ class ModuleImporter(private val program: Program, lines.asSequence() .mapIndexed { i, it -> i to it } .filter { (it.second as? Directive)?.directive == "%import" } - .forEach { executeImportDirective(it.second as Directive, modulePath) } + .forEach { executeImportDirective(it.second as Directive, moduleAst) } moduleAst.statements = lines return moduleAst } - private fun executeImportDirective(import: Directive, source: Path): Module? { + private fun executeImportDirective(import: Directive, importingModule: Module?): Module? { if(import.directive!="%import" || import.args.size!=1 || import.args[0].name==null) throw SyntaxError("invalid import directive", import.position) val moduleName = import.args[0].name!! @@ -90,19 +84,19 @@ class ModuleImporter(private val program: Program, val existing = program.modules.singleOrNull { it.name == moduleName } if(existing!=null) - return null + return null // TODO: why return null instead of Module instance? - val srcCode = tryGetModuleFromResource("$moduleName.p8", compilationTargetName) + var srcCode = tryGetModuleFromResource("$moduleName.p8", compilationTargetName) val importedModule = - if (srcCode != null) { // found in resources - // load the module from the embedded resource + if (srcCode != null) { println("importing '$moduleName' (library): ${srcCode.origin}") - val path = Path.of(URL(srcCode.origin).file) - importModule(srcCode.getCharStream(), path) - } else { - val modulePath = tryGetModuleFromFile(moduleName, source, import.position) - importModule(modulePath) - } + importModule(srcCode) + } else { + srcCode = tryGetModuleFromFile(moduleName, importingModule) + if (srcCode == null) + throw NoSuchFileException(File("$moduleName.p8")) + importModule(srcCode) + } removeDirectivesFromImportedModule(importedModule) return importedModule @@ -130,18 +124,27 @@ class ModuleImporter(private val program: Program, return null } - private fun tryGetModuleFromFile(name: String, source: Path, position: Position?): Path { + private fun tryGetModuleFromFile(name: String, importingModule: Module?): SourceCode? { val fileName = "$name.p8" - val libpaths = libdirs.map {Path.of(it)} + val libpaths = libdirs.map { Path.of(it) } val locations = - (if(source.toString().isEmpty()) libpaths else libpaths.drop(1) + listOf(source.parent ?: Path.of("."))) + - listOf(Paths.get(Paths.get("").toAbsolutePath().toString(), "prog8lib")) + if (importingModule == null) { // <=> imported from library module + libpaths + } else { + libpaths.drop(1) + // TODO: why drop the first? + // FIXME: won't work until Prog8Parser is fixed s.t. it fully initialzes the modules it returns + listOf(Path.of(importingModule.position.file).parent ?: Path.of(".")) + + listOf(Path.of(".", "prog8lib")) + } locations.forEach { - val file = pathFrom(it.toString(), fileName) - if (Files.isReadable(file)) return file + try { + return SourceCode.fromPath(it.resolve(fileName)) + } catch (e: NoSuchFileException) { + } } - throw ParsingFailedError("$position Import: no module source file '$fileName' found (I've looked in: embedded libs and $locations)") + //throw ParsingFailedError("$position Import: no module source file '$fileName' found (I've looked in: embedded libs and $locations)") + return null } } diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 45c684d60..96e341abe 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -4,7 +4,7 @@ import org.antlr.v4.runtime.* import prog8.ast.Module import prog8.ast.antlr.toAst import prog8.ast.base.Position -import java.nio.file.Path +import kotlin.io.path.Path open class ParsingFailedError(override var message: String) : Exception(message) @@ -41,7 +41,7 @@ object Prog8Parser { val parseTree = parser.module() val moduleName = "anonymous" - val module = parseTree.toAst(moduleName, pathFrom(""), PetsciiEncoding) + val module = parseTree.toAst(moduleName, Path(""), PetsciiEncoding) // TODO: use Module ctor directly for (statement in module.statements) { From d3e026d82a7e1ed5175b2b094198e40fdb6beb7d Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 19:58:06 +0200 Subject: [PATCH 12/68] +/* non-unique module names: provide more info, add TODO --- .../compiler/astprocessing/AstExtensions.kt | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index 2497e5d69..47579d965 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -58,8 +58,24 @@ internal fun Program.checkIdentifiers(errors: IErrorReporter, options: Compilati lit2decl.applyModifications() } - if (modules.map { it.name }.toSet().size != modules.size) { - throw FatalAstException("modules should all be unique") + // Check if each module has a unique name. + // If not report those that haven't. + // TODO: move check for unique module names to earlier stage and/or to unit tests + val namesToModules = mapOf>().toMutableMap() + for (m in modules) { + var others = namesToModules[m.name] + if (others == null) { + namesToModules.put(m.name, listOf(m).toMutableList()) + } else { + others.add(m) + } + } + val nonUniqueNames = namesToModules.keys + .map { Pair(it, namesToModules[it]!!.size) } + .filter { it.second > 1 } + .map { "\"${it.first}\" (x${it.second})"} + if (nonUniqueNames.size > 0) { + throw FatalAstException("modules must have unique names; of the ttl ${modules.size} these have not: $nonUniqueNames") } } From 4096aae8d42eab552c722297feb9593427cf4f8b Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 20:07:24 +0200 Subject: [PATCH 13/68] * SourceCode.toString() now states both, java class and .origin --- compilerAst/src/prog8/parser/SourceCode.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/compilerAst/src/prog8/parser/SourceCode.kt b/compilerAst/src/prog8/parser/SourceCode.kt index 02b3f721b..c8744d628 100644 --- a/compilerAst/src/prog8/parser/SourceCode.kt +++ b/compilerAst/src/prog8/parser/SourceCode.kt @@ -29,13 +29,9 @@ abstract class SourceCode() { /** * Deliberately does NOT return the actual text. - * Use [getCharStream]. + * For this - if at all - use [getCharStream]. */ - final override fun toString() = super.toString() - - - - + final override fun toString() = "${this.javaClass.name}[${this.origin}]" // "static" factory methods companion object { From 44da7a302fc20031a75bc408db8e35b941f75c1f Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 20:09:50 +0200 Subject: [PATCH 14/68] + temporarily hack together a module name inside Prog8Parser.parseModule, to make the current all-too-simple import resolution work --- compilerAst/src/prog8/parser/Prog8Parser.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 96e341abe..bc7182a4a 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -39,9 +39,19 @@ object Prog8Parser { parser.addErrorListener(antlrErrorListener) val parseTree = parser.module() - val moduleName = "anonymous" - val module = parseTree.toAst(moduleName, Path(""), PetsciiEncoding) + // FIXME: hacking together a name for the module: + var moduleName = src.origin + if (moduleName.startsWith(" Date: Sun, 4 Jul 2021 18:09:16 +0200 Subject: [PATCH 15/68] * fix (hack) .name, .source and .position of Modules from the parser (via temp. subclass ParsedModule) The temporary subclass ParsedModule : Module is introduced to concentrate all the workaround stuff in one place *while still not changing any public signature* such as of the Module ctor. The convention used to indicate stuff from resources is still "" not "@embedded@"- *note that this is caught by 3 tests in compiler* --- compilerAst/src/prog8/ast/AstToplevel.kt | 2 +- .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 4 +- compilerAst/src/prog8/parser/Prog8Parser.kt | 87 ++++++++++++------ compilerAst/test/TestProg8Parser.kt | 89 ++++++++++++++++--- 4 files changed, 141 insertions(+), 41 deletions(-) diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 216d0bcb5..1d57f8508 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -339,7 +339,7 @@ class Program(val name: String, } } -class Module(override val name: String, +open class Module(override val name: String, override var statements: MutableList, override val position: Position, val source: Path) : Node, INameScope { diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index 0cf6e7eac..d7abc240d 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -41,7 +41,7 @@ private fun ParserRuleContext.toPosition() : Position { return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine + stop.text.length) } -private fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Statement { +internal fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Block { val blockstatements = block_statement().map { when { it.variabledeclaration()!=null -> it.variabledeclaration().toAst(encoding) @@ -351,7 +351,7 @@ private fun Prog8ANTLRParser.DatatypeContext.toAst() = DataType.valueOf(text.upp private fun Prog8ANTLRParser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = ArrayIndex(expression().toAst(encoding), toPosition()) -private fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = +internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = Directive(directivename.text, directivearg().map { it.toAst() }, toPosition()) private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index bc7182a4a..35901b82e 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -4,6 +4,8 @@ import org.antlr.v4.runtime.* import prog8.ast.Module import prog8.ast.antlr.toAst import prog8.ast.base.Position +import prog8.ast.statements.Block +import prog8.ast.statements.Directive import kotlin.io.path.Path @@ -16,19 +18,10 @@ class ParseError(override var message: String, val position: Position, cause: Ru } } -private fun RecognitionException.getPosition(provenance: String) : Position { - val offending = this.offendingToken - val line = offending.line - val beginCol = offending.charPositionInLine - val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? - val pos = Position(provenance, line, beginCol, endCol) - return pos -} - object Prog8Parser { fun parseModule(src: SourceCode): Module { - val antlrErrorListener = AntlrErrorListener(src.origin) + val antlrErrorListener = AntlrErrorListener(src) val lexer = Prog8ANTLRLexer(src.getCharStream()) lexer.removeErrorListeners() lexer.addErrorListener(antlrErrorListener) @@ -40,24 +33,55 @@ object Prog8Parser { val parseTree = parser.module() - // FIXME: hacking together a name for the module: - var moduleName = src.origin - if (moduleName.startsWith("") + + private class ParsedModule(src: SourceCode) : Module( + // FIXME: hacking together a name for the module: + name = src.pathString() + .substringBeforeLast(".") + .substringAfterLast("/") + .substringAfterLast("\\") + .replace("String@", "anonymous_"), + // FIXME: hacking together a path + source = Path(src.pathString()), + statements = mutableListOf(), + position = Position(src.origin, 1, 0, 0) + ) { + val provenance = Pair(src, Triple(1, 0, 0)) + + /** + * Adds a [Directive] to [statements] and + * sets this Module as its [parent]. + * Note: you can only add [Directive]s or [Block]s to a Module. + */ + fun add(child: Directive) { + child.linkParents(this) + statements.add(child) + } + /** + * Adds a [Block] to [statements] and + * sets this Module as its [parent]. + * Note: you can only add [Directive]s or [Block]s to a Module. + */ + fun add(child: Block) { + child.linkParents(this) + statements.add(child) + } } private object Prog8ErrorStrategy: BailErrorStrategy() { @@ -86,15 +110,24 @@ object Prog8Parser { } } - private class AntlrErrorListener(val sourceCodeProvenance: String): BaseErrorListener() { + private class AntlrErrorListener(val src: SourceCode): BaseErrorListener() { override fun syntaxError(recognizer: Recognizer<*, *>?, offendingSymbol: Any?, line: Int, charPositionInLine: Int, msg: String, e: RecognitionException?) { if (e == null) { TODO("no RecognitionException - create your own ParseError") //throw ParseError() } else { - throw ParseError(msg, e.getPosition(sourceCodeProvenance), e) + throw ParseError(msg, e.getPosition(src.origin), e) } } } + private fun RecognitionException.getPosition(file: String) : Position { + val offending = this.offendingToken + val line = offending.line + val beginCol = offending.charPositionInLine + val endCol = beginCol + offending.stopIndex - offending.startIndex // TODO: point to col *after* token? + val pos = Position(file, line, beginCol, endCol) + return pos + } + } diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 381feffa6..a780643cc 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,16 +1,27 @@ package prog8tests import org.junit.jupiter.api.Test +import prog8.ast.Node +import prog8.ast.base.Position import kotlin.test.* import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* import prog8.ast.statements.Block +import prog8.ast.statements.Directive import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode class TestProg8Parser { + val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! + val fixturesDir = workingDir.resolve("test/fixtures") + + @Test + fun testDirectoriesSanityCheck() { + assertEquals("compilerAst", workingDir.fileName.toString()) + assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") + } @Test fun testModuleSourceNeedNotEndWithNewline() { @@ -165,6 +176,51 @@ class TestProg8Parser { assertEquals(1, module.statements.size) } + + @Test + fun testModuleNameForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + + // Note: assertContains has *actual* as first param + assertContains(module.name, Regex("^anonymous_[0-9a-f]+$")) + } + + @Test + fun testModuleNameForSourceFromPath() { + val path = fixturesDir.resolve("simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + + assertEquals(path.nameWithoutExtension, module.name) + } + + + fun assertPosition(actual: Position, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) { + assertEquals(expLine, actual.line, ".position.line (1-based)") + assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + assertEquals(expFile, actual.file, ".position.file") + } + + fun assertPosition(actual: Position, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) { + assertEquals(expLine, actual.line, ".position.line (1-based)") + assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + // Note: assertContains expects *actual* value first + assertContains(actual.file, expFile, ".position.file") + } + + fun assertPositionOf(actual: Node, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + fun assertPositionOf(actual: Node, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + @Test fun testErrorLocationForSourceFromString() { val srcText = "bad * { }\n" @@ -173,30 +229,41 @@ class TestProg8Parser { try { parseModule(SourceCode.of(srcText)) } catch (e: ParseError) { - // Note: assertContains expects *actual* value first - assertContains(e.position.file, Regex("^$")) - assertEquals(1, e.position.line, "line; should be 1-based") - assertEquals(4, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(4, e.position.endCol, "endCol; should be 0-based") + assertPosition(e.position, Regex("^$"), 1, 4, 4) } } @Test fun testErrorLocationForSourceFromPath() { - val filename = "file_with_syntax_error.p8" - val path = Path.of("test", "fixtures", filename) + val path = fixturesDir.resolve("file_with_syntax_error.p8") assertFailsWith { parseModule(SourceCode.fromPath(path)) } try { parseModule(SourceCode.fromPath(path)) } catch (e: ParseError) { - assertEquals(path.absolutePathString(), e.position.file, "provenance; should be the path's filename, incl. extension '.p8'") - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + assertPosition(e.position, path.absolutePathString(), 2, 6, 6) } } + @Test + fun testModulePositionForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + assertPositionOf(module, Regex("^$"), 1, 0, 0) + } + + @Test + fun testModulePositionForSourceFromPath() { + val path = fixturesDir.resolve("simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + assertPositionOf(module, path.absolutePathString(), 1, 0, 0) + } + + @Test fun testProg8Ast() { val module = parseModule(SourceCode.of(""" From b0073ac933ca42e2eb9fa3a5020dc78740125c86 Mon Sep 17 00:00:00 2001 From: meisl Date: Fri, 9 Jul 2021 16:28:04 +0200 Subject: [PATCH 16/68] * used "@embedded@" convention instead of "", put it into SourceCode --- compilerAst/src/prog8/parser/Prog8Parser.kt | 11 ++-------- compilerAst/src/prog8/parser/SourceCode.kt | 24 +++++++++++++++++---- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 35901b82e..0cde17230 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -41,19 +41,12 @@ object Prog8Parser { parseTree.block().forEach { module.add(it.toAst(module.isLibrary(), PetsciiEncoding)) } return module - } - - // FIXME: hacking together a path string: - private fun SourceCode.pathString() = - origin - .substringAfter("") + } private class ParsedModule(src: SourceCode) : Module( // FIXME: hacking together a name for the module: name = src.pathString() - .substringBeforeLast(".") + .substringBeforeLast(".") // must also work with an origin = "" .substringAfterLast("/") .substringAfterLast("\\") .replace("String@", "anonymous_"), diff --git a/compilerAst/src/prog8/parser/SourceCode.kt b/compilerAst/src/prog8/parser/SourceCode.kt index c8744d628..005d90fbf 100644 --- a/compilerAst/src/prog8/parser/SourceCode.kt +++ b/compilerAst/src/prog8/parser/SourceCode.kt @@ -10,16 +10,32 @@ import kotlin.io.path.* * Encapsulates - and ties together - actual source code (=text) * and its [origin]. */ -abstract class SourceCode() { +abstract class SourceCode { + + /** + * To be used *only* by the parser (as input to a TokenStream). + * DO NOT mess around with! + */ + internal abstract fun getCharStream(): CharStream + /** * Where this [SourceCode] instance came from. * This can be one of the following: * * a normal string representation of a [java.nio.file.Path], if it originates from a file (see [fromPath]) * * `` if was created via [of] - * * `` if it came from resources (see [fromResources]) + * * `@embedded@/x/y/z.ext` if it came from resources (see [fromResources]) */ abstract val origin: String - abstract fun getCharStream(): CharStream + + + /** + * FIXME: hacking together a [SourceCode]'s "path string" + * This is really just [origin] with any stuff removed that would render it an invalid path name. + * (Note: a *valid* path name does NOT mean that the denoted file or folder *exists*) + */ + fun pathString() = + origin + .substringAfter("<").substringBeforeLast(">") // or from plain string? /** * The source code as plain string. @@ -92,7 +108,7 @@ abstract class SourceCode() { reason = "looked in resources rooted at $rscRoot") } return object : SourceCode() { - override val origin = "" + override val origin = "@embedded@$normalized" override fun getCharStream(): CharStream { val inpStr = object{}.javaClass.getResourceAsStream(normalized) val chars = CharStreams.fromStream(inpStr) From 19bb56df47e358201925e9caa29d68bd3aa79ba0 Mon Sep 17 00:00:00 2001 From: meisl Date: Fri, 9 Jul 2021 17:32:33 +0200 Subject: [PATCH 17/68] * no more scattering magic "@embedded@" all over the place: add SourceCode.isFromResources, *change Module.source from type Path to type SourceCode* --- compiler/src/prog8/compiler/Compiler.kt | 15 +++++++++++---- .../compiler/astprocessing/AstChecker.kt | 19 ++++++++++++++++--- .../compiler/target/cpu6502/codegen/AsmGen.kt | 8 ++++++-- compiler/test/AsmgenTests.kt | 2 +- compiler/test/TestMemory.kt | 14 +++++++------- compilerAst/src/prog8/ast/AstToplevel.kt | 15 ++++----------- .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 5 +++-- compilerAst/src/prog8/parser/Prog8Parser.kt | 12 +++++------- compilerAst/src/prog8/parser/SourceCode.kt | 9 +++++++++ 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index c256f4850..5cb3cc0e8 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -22,6 +22,7 @@ import prog8.parser.moduleName import java.io.File import java.io.InputStream import java.nio.file.Path +import kotlin.io.path.Path import kotlin.system.measureTimeMillis @@ -183,7 +184,10 @@ private fun parseImports(filepath: Path, importer.importModule(filepath) errors.report() - val importedFiles = programAst.modules.filter { !it.source.startsWith("@embedded@") }.map { it.source } + val importedFiles = programAst.modules + .mapNotNull { it.source } + .filter { !it.isFromResources } // TODO: parseImports/importedFiles - maybe rather `source.isFromFilesystem`? + .map { Path(it.pathString()) } val compilerOptions = determineCompilationOptions(programAst, compTarget) if (compilerOptions.launcher == LauncherType.BASIC && compilerOptions.output != OutputType.PRG) throw ParsingFailedError("${programAst.modules.first().position} BASIC launcher requires output type PRG.") @@ -357,14 +361,14 @@ fun printAst(programAst: Program) { println() } -fun loadAsmIncludeFile(filename: String, source: Path): String { - return if (filename.startsWith("library:")) { +fun loadAsmIncludeFile(filename: String, sourcePath: Path): String { + return if (filename.startsWith("library:")) { // FIXME: is the prefix "library:" or is it "@embedded@"? val resource = tryGetEmbeddedResource(filename.substring(8)) ?: throw IllegalArgumentException("library file '$filename' not found") resource.bufferedReader().use { it.readText() } } else { // first try in the isSameAs folder as where the containing file was imported from - val sib = source.resolveSibling(filename) + val sib = sourcePath.resolveSibling(filename) if (sib.toFile().isFile) sib.toFile().readText() else @@ -372,6 +376,9 @@ fun loadAsmIncludeFile(filename: String, source: Path): String { } } +/** + * Handle via SourceCode + */ internal fun tryGetEmbeddedResource(name: String): InputStream? { return object{}.javaClass.getResourceAsStream("/prog8lib/$name") } diff --git a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt index ea184b71f..20c9a830f 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt @@ -17,6 +17,7 @@ import prog8.compiler.target.Cx16Target import prog8.compiler.target.ICompilationTarget import java.io.CharConversionException import java.io.File +import kotlin.io.path.* import java.util.* internal class AstChecker(private val program: Program, @@ -728,11 +729,23 @@ internal class AstChecker(private val program: Program, } private fun checkFileExists(directive: Directive, filename: String) { - var definingModule = directive.parent + if (File(filename).isFile) + return + + var definingModule = directive.parent // TODO: why not just use directive.definingModule() here? while (definingModule !is Module) definingModule = definingModule.parent - if (!(filename.startsWith("library:") || definingModule.source.resolveSibling(filename).toFile().isFile || File(filename).isFile)) - errors.err("included file not found: $filename", directive.position) + if (definingModule.isLibrary()) + return + + val s = definingModule.source?.pathString() + if (s != null) { + val sourceFileCandidate = Path(s).resolveSibling(filename).toFile() + if (sourceFileCandidate.isFile) + return + } + + errors.err("included file not found: $filename", directive.position) } override fun visit(array: ArrayLiteralValue) { diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt index 212911571..cc55baed0 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt @@ -1311,13 +1311,17 @@ $repeatLabel lda $counterVar private fun translate(stmt: Directive) { when(stmt.directive) { "%asminclude" -> { - val sourcecode = loadAsmIncludeFile(stmt.args[0].str!!, stmt.definingModule().source) + // TODO: handle %asminclude with SourceCode + val sourcePath = Path.of(stmt.definingModule().source!!.pathString()) // FIXME: %asminclude inside non-library, non-filesystem module + val sourcecode = loadAsmIncludeFile(stmt.args[0].str!!, sourcePath) assemblyLines.add(sourcecode.trimEnd().trimStart('\n')) } "%asmbinary" -> { val offset = if(stmt.args.size>1) ", ${stmt.args[1].int}" else "" val length = if(stmt.args.size>2) ", ${stmt.args[2].int}" else "" - val includedSourcePath = stmt.definingModule().source.resolveSibling(stmt.args[0].str) + // TODO: handle %asmbinary with SourceCode + val sourcePath = Path.of(stmt.definingModule().source!!.pathString()) // FIXME: %asmbinary inside non-library, non-filesystem module + val includedSourcePath = sourcePath.resolveSibling(stmt.args[0].str) val relPath = Paths.get("").relativize(includedSourcePath) out(" .binary \"$relPath\" $offset $length") } diff --git a/compiler/test/AsmgenTests.kt b/compiler/test/AsmgenTests.kt index 8deeae874..20a77c71d 100644 --- a/compiler/test/AsmgenTests.kt +++ b/compiler/test/AsmgenTests.kt @@ -74,7 +74,7 @@ locallabel: val varInBlock = VarDecl(VarDeclType.VAR, DataType.UWORD, ZeropageWish.DONTCARE, null, "var_outside", null, false, false, false, Position.DUMMY) val block = Block("main", null, mutableListOf(labelInBlock, varInBlock, subroutine), false, Position.DUMMY) - val module = Module("test", mutableListOf(block), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(block), Position.DUMMY, null) module.linkParents(ParentSentinel) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.program = program diff --git a/compiler/test/TestMemory.kt b/compiler/test/TestMemory.kt index b9ca856d5..d0e926aa6 100644 --- a/compiler/test/TestMemory.kt +++ b/compiler/test/TestMemory.kt @@ -98,7 +98,7 @@ class TestMemory { val target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) module.linkParents(ParentSentinel) return target } @@ -117,7 +117,7 @@ class TestMemory { val target = AssignTarget(IdentifierReference(listOf("address"), Position.DUMMY), null, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) @@ -130,7 +130,7 @@ class TestMemory { val target = AssignTarget(IdentifierReference(listOf("address"), Position.DUMMY), null, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) @@ -143,7 +143,7 @@ class TestMemory { val target = AssignTarget(IdentifierReference(listOf("address"), Position.DUMMY), null, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertFalse(C64Target.isInRegularRAM(target, program)) @@ -156,7 +156,7 @@ class TestMemory { val target = AssignTarget(null, arrayindexed, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) @@ -170,7 +170,7 @@ class TestMemory { val target = AssignTarget(null, arrayindexed, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) @@ -184,7 +184,7 @@ class TestMemory { val target = AssignTarget(null, arrayindexed, null, Position.DUMMY) val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) - val module = Module("test", mutableListOf(subroutine), Position.DUMMY, Path.of("")) + val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) module.linkParents(ParentSentinel) assertFalse(C64Target.isInRegularRAM(target, program)) diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 1d57f8508..81436cdac 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -5,6 +5,7 @@ import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstVisitor +import prog8.parser.SourceCode import java.nio.file.Path import java.nio.file.Paths import kotlin.io.path.name @@ -265,7 +266,7 @@ class Program(val name: String, init { // insert a container module for all interned strings later if(modules.firstOrNull()?.name != internedStringsModuleName) { - val internedStringsModule = Module(internedStringsModuleName, mutableListOf(), Position.DUMMY, Path.of("")) + val internedStringsModule = Module(internedStringsModuleName, mutableListOf(), Position.DUMMY, null) modules.add(0, internedStringsModule) val block = Block(internedStringsModuleName, null, mutableListOf(), true, Position.DUMMY) internedStringsModule.statements.add(block) @@ -342,7 +343,7 @@ class Program(val name: String, open class Module(override val name: String, override var statements: MutableList, override val position: Position, - val source: Path) : Node, INameScope { + val source: SourceCode?) : Node, INameScope { override lateinit var parent: Node lateinit var program: Program @@ -370,15 +371,7 @@ open class Module(override val name: String, fun accept(visitor: IAstVisitor) = visitor.visit(this) fun accept(visitor: AstWalker, parent: Node) = visitor.visit(this, parent) - companion object { - fun pathForResource(resourcePath: String): Path { - return Paths.get("@embedded@/$resourcePath") - } - - fun isLibrary(source: Path) = source.name=="" || source.startsWith("@embedded@/") - } - - fun isLibrary() = isLibrary(source) + fun isLibrary() = (source == null) || source.isFromResources } diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index d7abc240d..1792215b2 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -9,6 +9,7 @@ import prog8.ast.base.* import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.parser.Prog8ANTLRParser +import prog8.parser.SourceCode import java.io.CharConversionException import java.io.File import java.nio.file.Path @@ -18,10 +19,10 @@ import java.nio.file.Path private data class NumericLiteral(val number: Number, val datatype: DataType) -internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { +internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: SourceCode, encoding: IStringEncoding) : Module { val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name val directives = this.directive().map { it.toAst() } - val blocks = this.block().map { it.toAst(Module.isLibrary(source), encoding) } + val blocks = this.block().map { it.toAst(isInLibrary = source.isFromResources, encoding) } return Module(nameWithoutSuffix, (directives + blocks).toMutableList(), toPosition(), source) } diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 0cde17230..4d080a8ea 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -6,7 +6,6 @@ import prog8.ast.antlr.toAst import prog8.ast.base.Position import prog8.ast.statements.Block import prog8.ast.statements.Directive -import kotlin.io.path.Path open class ParsingFailedError(override var message: String) : Exception(message) @@ -43,19 +42,18 @@ object Prog8Parser { return module } - private class ParsedModule(src: SourceCode) : Module( + private class ParsedModule(source: SourceCode) : Module( // FIXME: hacking together a name for the module: - name = src.pathString() + name = source.pathString() .substringBeforeLast(".") // must also work with an origin = "" .substringAfterLast("/") .substringAfterLast("\\") .replace("String@", "anonymous_"), - // FIXME: hacking together a path - source = Path(src.pathString()), statements = mutableListOf(), - position = Position(src.origin, 1, 0, 0) + position = Position(source.origin, 1, 0, 0), + source ) { - val provenance = Pair(src, Triple(1, 0, 0)) + val provenance = Pair(source, Triple(1, 0, 0)) /** * Adds a [Directive] to [statements] and diff --git a/compilerAst/src/prog8/parser/SourceCode.kt b/compilerAst/src/prog8/parser/SourceCode.kt index 005d90fbf..44f4fdb25 100644 --- a/compilerAst/src/prog8/parser/SourceCode.kt +++ b/compilerAst/src/prog8/parser/SourceCode.kt @@ -18,6 +18,12 @@ abstract class SourceCode { */ internal abstract fun getCharStream(): CharStream + /** + * Whether this [SourceCode] instance was created by + * factory method [fromResources] + */ + abstract val isFromResources: Boolean + /** * Where this [SourceCode] instance came from. * This can be one of the following: @@ -58,6 +64,7 @@ abstract class SourceCode { */ fun of(text: String): SourceCode { return object : SourceCode() { + override val isFromResources = false override val origin = "" override fun getCharStream(): CharStream { return CharStreams.fromString(text) @@ -86,6 +93,7 @@ abstract class SourceCode { throw AccessDeniedException(path.toFile(), reason = "Is not readable") val normalized = path.normalize() return object : SourceCode() { + override val isFromResources = false override val origin = normalized.absolutePathString() override fun getCharStream(): CharStream { return CharStreams.fromPath(normalized) @@ -108,6 +116,7 @@ abstract class SourceCode { reason = "looked in resources rooted at $rscRoot") } return object : SourceCode() { + override val isFromResources = true override val origin = "@embedded@$normalized" override fun getCharStream(): CharStream { val inpStr = object{}.javaClass.getResourceAsStream(normalized) From 7530fb67c89280d7054fc695060e9f8dac3e3e78 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 4 Jul 2021 18:09:16 +0200 Subject: [PATCH 18/68] + add tests for inner nodes' positions; refactor tests --- compilerAst/test/TestProg8Parser.kt | 96 +++++++++++++++++++----- compilerAst/test/fixtures/simple_main.p8 | 4 +- 2 files changed, 78 insertions(+), 22 deletions(-) diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index a780643cc..761d66461 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -3,11 +3,10 @@ package prog8tests import org.junit.jupiter.api.Test import prog8.ast.Node import prog8.ast.base.Position +import prog8.ast.statements.* import kotlin.test.* import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* -import prog8.ast.statements.Block -import prog8.ast.statements.Directive import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode @@ -199,25 +198,27 @@ class TestProg8Parser { } - fun assertPosition(actual: Position, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) { - assertEquals(expLine, actual.line, ".position.line (1-based)") - assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) - assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") - assertEquals(expFile, actual.file, ".position.file") + fun assertPosition(actual: Position, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) { + require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) + if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") + if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + if (expFile != null) assertEquals(expFile, actual.file, ".position.file") } - fun assertPosition(actual: Position, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) { - assertEquals(expLine, actual.line, ".position.line (1-based)") - assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) - assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + fun assertPosition(actual: Position, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) { + require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) + if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") + if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) + if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") // Note: assertContains expects *actual* value first - assertContains(actual.file, expFile, ".position.file") + if (expFile != null) assertContains(actual.file, expFile, ".position.file") } - fun assertPositionOf(actual: Node, expFile: String, expLine: Int, expStartCol: Int, expEndCol: Int) = + fun assertPositionOf(actual: Node, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) - fun assertPositionOf(actual: Node, expFile: Regex, expLine: Int, expStartCol: Int, expEndCol: Int) = + fun assertPositionOf(actual: Node, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) @@ -230,7 +231,7 @@ class TestProg8Parser { parseModule(SourceCode.of(srcText)) } catch (e: ParseError) { assertPosition(e.position, Regex("^$"), 1, 4, 4) - } + } } @Test @@ -241,7 +242,7 @@ class TestProg8Parser { try { parseModule(SourceCode.fromPath(path)) } catch (e: ParseError) { - assertPosition(e.position, path.absolutePathString(), 2, 6, 6) + assertPosition(e.position, path.absolutePathString(), 2, 6) // TODO: endCol wrong } } @@ -252,7 +253,7 @@ class TestProg8Parser { } """.trimIndent() val module = parseModule(SourceCode.of(srcText)) - assertPositionOf(module, Regex("^$"), 1, 0, 0) + assertPositionOf(module, Regex("^$"), 1, 0) // TODO: endCol wrong } @Test @@ -260,10 +261,65 @@ class TestProg8Parser { val path = fixturesDir.resolve("simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) - assertPositionOf(module, path.absolutePathString(), 1, 0, 0) + assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong } - + @Test + fun testInnerNodePositionsForSourceFromPath() { + val path = fixturesDir.resolve("simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + val mpf = module.position.file + + assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong + val mainBlock = module.statements.filterIsInstance()[0] + assertPositionOf(mainBlock, mpf, 1, 0) // TODO: endCol wrong! + val startSub = mainBlock.statements.filterIsInstance()[0] + assertPositionOf(startSub, mpf, 2, 4) // TODO: endCol wrong! + } + + /** + * TODO: this test is testing way too much at once + */ + @Test + fun testInnerNodePositionsForSourceFromString() { + val srcText = """ + %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? + main { + sub start() { + ubyte foo = 42 + ubyte bar + when (foo) { + 23 -> bar = 'x' ; WhenChoice, also directly inheriting Node + 42 -> bar = 'y' + else -> bar = 'z' + } + } + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + val mpf = module.position.file + + val targetDirective = module.statements.filterIsInstance()[0] + assertPositionOf(targetDirective, mpf, 1, 0) // TODO: endCol wrong! + val mainBlock = module.statements.filterIsInstance()[0] + assertPositionOf(mainBlock, mpf, 2, 0) // TODO: endCol wrong! + val startSub = mainBlock.statements.filterIsInstance()[0] + assertPositionOf(startSub, mpf, 3, 4) // TODO: endCol wrong! + val declFoo = startSub.statements.filterIsInstance()[0] + assertPositionOf(declFoo, mpf, 4, 8) // TODO: endCol wrong! + val rhsFoo = declFoo.value!! + assertPositionOf(rhsFoo, mpf, 4, 20) // TODO: endCol wrong! + val declBar = startSub.statements.filterIsInstance()[1] + assertPositionOf(declBar, mpf, 5, 8) // TODO: endCol wrong! + val whenStmt = startSub.statements.filterIsInstance()[0] + assertPositionOf(whenStmt, mpf, 6, 8) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[0], mpf, 7, 12) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[1], mpf, 8, 12) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[2], mpf, 9, 12) // TODO: endCol wrong! + } + + @Test fun testProg8Ast() { val module = parseModule(SourceCode.of(""" @@ -274,6 +330,6 @@ class TestProg8Parser { } """)) assertIs(module.statements.first()) - assertEquals((module.statements.first() as Block).name, "main") + assertEquals("main", (module.statements.first() as Block).name) } } diff --git a/compilerAst/test/fixtures/simple_main.p8 b/compilerAst/test/fixtures/simple_main.p8 index afaa79f93..fb81add50 100644 --- a/compilerAst/test/fixtures/simple_main.p8 +++ b/compilerAst/test/fixtures/simple_main.p8 @@ -1,4 +1,4 @@ main { - sub start() { - } + sub start() { + } } From c3e9d4a9f8c95710da5672484df31a2ff2bf7fbf Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 10 Jul 2021 20:50:07 +0200 Subject: [PATCH 19/68] * make resources available in compilerAst/test s; *you may have to re-import the gradle project into IDEA* Note: these resources are NOT going into the production .jar --- compilerAst/build.gradle | 8 ++++---- compilerAst/compilerAst.iml | 1 + compilerAst/res/prog8lib/math.asm | 20 ++++++++++++++++++++ compilerAst/res/prog8lib/math.p8 | 7 +++++++ 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 compilerAst/res/prog8lib/math.asm create mode 100644 compilerAst/res/prog8lib/math.p8 diff --git a/compilerAst/build.gradle b/compilerAst/build.gradle index 78f7d8fd3..4fa87b242 100644 --- a/compilerAst/build.gradle +++ b/compilerAst/build.gradle @@ -45,15 +45,15 @@ sourceSets { main { java { srcDirs = ["${project.projectDir}/src"] - } - resources { - srcDirs = ["${project.projectDir}/res"] - } + } } test { java { srcDirs = ["${project.projectDir}/test"] } + resources { + srcDirs = ["${project.projectDir}/res"] + } } } diff --git a/compilerAst/compilerAst.iml b/compilerAst/compilerAst.iml index 240433064..24edf66c1 100644 --- a/compilerAst/compilerAst.iml +++ b/compilerAst/compilerAst.iml @@ -3,6 +3,7 @@ + diff --git a/compilerAst/res/prog8lib/math.asm b/compilerAst/res/prog8lib/math.asm new file mode 100644 index 000000000..2638821fb --- /dev/null +++ b/compilerAst/res/prog8lib/math.asm @@ -0,0 +1,20 @@ +; just for tests - DISFUNCTIONAL! + + +math_store_reg .byte 0 ; temporary storage + + +multiply_bytes .proc + ; -- multiply 2 bytes A and Y, result as byte in A (signed or unsigned) + sta P8ZP_SCRATCH_B1 ; num1 + sty P8ZP_SCRATCH_REG ; num2 + lda #0 + beq _enterloop +_doAdd clc + adc P8ZP_SCRATCH_B1 +_loop asl P8ZP_SCRATCH_B1 +_enterloop lsr P8ZP_SCRATCH_REG + bcs _doAdd + bne _loop + rts + .pend diff --git a/compilerAst/res/prog8lib/math.p8 b/compilerAst/res/prog8lib/math.p8 new file mode 100644 index 000000000..234079159 --- /dev/null +++ b/compilerAst/res/prog8lib/math.p8 @@ -0,0 +1,7 @@ +; Internal Math library routines - always included by the compiler +; +; Written by Irmen de Jong (irmen@razorvine.net) - license: GNU GPL 3.0 + +math { + %asminclude "library:math.asm" +} From ddaef3e5d5ea5475180d2a9b8464325f554a830e Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 10 Jul 2021 20:55:23 +0200 Subject: [PATCH 20/68] + add tests for SourceCode.fromResources; refactor tests --- compiler/test/TestCompilerOnCharLit.kt | 11 +-- compilerAst/test/TestSourceCode.kt | 118 ++++++++++++++++++++++--- 2 files changed, 112 insertions(+), 17 deletions(-) diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index 3c6fcb600..43344200e 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -2,6 +2,9 @@ package prog8tests import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import kotlin.io.path.* +import kotlin.test.* + import prog8.ast.IFunctionCall import prog8.ast.base.DataType import prog8.ast.base.VarDeclType @@ -9,12 +12,6 @@ import prog8.ast.expressions.IdentifierReference import prog8.ast.expressions.NumericLiteralValue import prog8.compiler.compileProgram import prog8.compiler.target.Cx16Target -import kotlin.io.path.Path -import kotlin.io.path.absolute -import kotlin.io.path.isDirectory -import kotlin.test.assertEquals -import kotlin.test.assertIs -import kotlin.test.assertTrue /** @@ -29,7 +26,7 @@ class TestCompilerOnCharLit { val outputDir = workingDir.resolve("build/tmp/test") @Test - fun testDirectoriesSanityCheck() { + fun sanityCheckDirectories() { assertEquals("compiler", workingDir.fileName.toString()) assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") diff --git a/compilerAst/test/TestSourceCode.kt b/compilerAst/test/TestSourceCode.kt index 7a20767f2..22e54cc12 100644 --- a/compilerAst/test/TestSourceCode.kt +++ b/compilerAst/test/TestSourceCode.kt @@ -2,8 +2,8 @@ package prog8tests import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.assertThrows import kotlin.test.* -import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* import prog8.parser.SourceCode @@ -11,6 +11,19 @@ import prog8.parser.SourceCode @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestSourceCode { + val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! + val fixturesDir = workingDir.resolve("test/fixtures") + val resourcesDir = workingDir.resolve("res") + val outputDir = workingDir.resolve("build/tmp/test") + + @Test + fun sanityCheckDirectories() { + assertEquals("compilerAst", workingDir.fileName.toString()) + assertTrue(workingDir.isDirectory(), "sanity check; should be directory: $workingDir") + assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") + assertTrue(resourcesDir.isDirectory(), "sanity check; should be directory: $resourcesDir") + assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + } @Test fun testFactoryMethod_Of() { @@ -27,7 +40,7 @@ class TestSourceCode { @Test fun testFromPathWithNonExistingPath() { val filename = "i_do_not_exist.p8" - val path = Path.of("test", "fixtures", filename) + val path = fixturesDir.resolve(filename) assertFalse(path.exists(), "sanity check: file should not exist: ${path.absolute()}") assertFailsWith { SourceCode.fromPath(path) } @@ -35,8 +48,8 @@ class TestSourceCode { @Test fun testFromPathWithMissingExtension_p8() { - val pathWithoutExt = Path.of("test", "fixtures", "simple_main") - val pathWithExt = Path.of(pathWithoutExt.toString() + ".p8") + val pathWithoutExt = fixturesDir.resolve("simple_main") + val pathWithExt = Path(pathWithoutExt.toString() + ".p8") assertTrue(pathWithExt.isRegularFile(), "sanity check: should be normal file: ${pathWithExt.absolute()}") assertTrue(pathWithExt.isReadable(), "sanity check: should be readable: ${pathWithExt.absolute()}") @@ -45,16 +58,13 @@ class TestSourceCode { @Test fun testFromPathWithDirectory() { - val path = Path.of("test", "fixtures") - - assertTrue(path.isDirectory(), "sanity check: should be a directory") - assertFailsWith { SourceCode.fromPath(path) } + assertFailsWith { SourceCode.fromPath(fixturesDir) } } @Test fun testFromPathWithExistingPath() { val filename = "simple_main.p8" - val path = Path.of("test", "fixtures", filename) + val path = fixturesDir.resolve(filename) val src = SourceCode.fromPath(path) val expectedOrigin = path.normalize().absolutePathString() @@ -68,7 +78,7 @@ class TestSourceCode { @Test fun testFromPathWithExistingNonNormalizedPath() { val filename = "simple_main.p8" - val path = Path.of(".", "test", "..", "test", "fixtures", filename) + val path = Path(".", "test", "..", "test", "fixtures", filename) val src = SourceCode.fromPath(path) val expectedOrigin = path.normalize().absolutePathString() @@ -79,4 +89,92 @@ class TestSourceCode { assertEquals(expectedSrcText, actualSrcText) } + @Test + fun testFromResourcesWithExistingP8File_withoutLeadingSlash() { + val pathString = "prog8lib/math.p8" + val src = SourceCode.fromResources(pathString) + + assertEquals("@embedded@/$pathString", src.origin) + + val expectedSrcText = resourcesDir.resolve(pathString).toFile().readText() + val actualSrcText = src.asString() + assertEquals(expectedSrcText, actualSrcText) + } + + @Test + fun testFromResourcesWithExistingP8File_withLeadingSlash() { + val pathString = "/prog8lib/math.p8" + val src = SourceCode.fromResources(pathString) + + assertEquals("@embedded@$pathString", src.origin) + + val expectedSrcText = resourcesDir.resolve(pathString.substringAfter("/")).toFile().readText() + val actualSrcText = src.asString() + assertEquals(expectedSrcText, actualSrcText) + } + + @Test + fun testFromResourcesWithExistingAsmFile_withoutLeadingSlash() { + val pathString = "prog8lib/math.asm" + val src = SourceCode.fromResources(pathString) + + assertEquals("@embedded@/$pathString", src.origin) + + val expectedSrcText = resourcesDir.resolve(pathString).toFile().readText() + val actualSrcText = src.asString() + assertEquals(expectedSrcText, actualSrcText) + assertTrue(src.isFromResources, ".isFromResources") + } + + @Test + fun testFromResourcesWithExistingAsmFile_withLeadingSlash() { + val pathString = "/prog8lib/math.asm" + val src = SourceCode.fromResources(pathString) + + assertEquals("@embedded@$pathString", src.origin) + + val expectedSrcText = resourcesDir.resolve(pathString.substringAfter("/")).toFile().readText() + val actualSrcText = src.asString() + assertEquals(expectedSrcText, actualSrcText) + } + + @Test + fun testFromResourcesWithNonNormalizedPath() { + val pathString = "/prog8lib/../prog8lib/math.p8" + val src = SourceCode.fromResources(pathString) + + assertEquals("@embedded@/prog8lib/math.p8", src.origin) + + val expectedSrcText = Path( "res", pathString).toFile().readText() + val actualSrcText = src.asString() + assertEquals(expectedSrcText, actualSrcText) + assertTrue(src.isFromResources, ".isFromResources") + } + + + @Test + fun testFromResourcesWithNonExistingFile_withLeadingSlash() { + val pathString = "/prog8lib/i_do_not_exist" + val resPath = resourcesDir.resolve(pathString.substringAfter("/")) + assertFalse(resPath.exists(), "sanity check: should not exist: $resPath") + assertThrows { SourceCode.fromResources(pathString) } + } + @Test + fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() { + val pathString = "prog8lib/i_do_not_exist" + val resPath = resourcesDir.resolve(pathString) + assertFalse(resPath.exists(), "sanity check: should not exist: $resPath") + assertThrows { SourceCode.fromResources(pathString) } + } + + /** + * TODO("inside resources: cannot tell apart a folder from a file") + */ + //@Test + fun testFromResourcesWithDirectory() { + val pathString = "/prog8lib" + assertThrows { SourceCode.fromResources(pathString) } + } + + } From 6fa50a699f40f1bf383feea7858a15f9207a8129 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 10 Jul 2021 21:03:14 +0200 Subject: [PATCH 21/68] + add two tests for parseModule with empty source text (from File and from String) --- compilerAst/test/TestProg8Parser.kt | 15 +++++++++++++++ compilerAst/test/fixtures/empty.p8 | 0 2 files changed, 15 insertions(+) create mode 100644 compilerAst/test/fixtures/empty.p8 diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 761d66461..202464f5c 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -176,6 +176,21 @@ class TestProg8Parser { } + @Test + fun testParseModuleWithEmptyString() { + val module = parseModule(SourceCode.of("")) + assertEquals(0, module.statements.size) + } + + @Test + fun testParseModuleWithEmptyFile() { + val path = fixturesDir.resolve("empty.p8") + assertTrue(path.isRegularFile(), "sanity check: should be regular file: $path") + + val module = parseModule(SourceCode.fromPath(path)) + assertEquals(0, module.statements.size) + } + @Test fun testModuleNameForSourceFromString() { val srcText = """ diff --git a/compilerAst/test/fixtures/empty.p8 b/compilerAst/test/fixtures/empty.p8 new file mode 100644 index 000000000..e69de29bb From 39d5b7edb045712b5ef9304da2c43bd49a33abe7 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 10 Jul 2021 10:30:29 +0200 Subject: [PATCH 22/68] + test examples for both platforms, cx16 and c64; test two more: tehtriz and textelite (the largest ones, 20KB / 36KB) --- compiler/test/TestCompilerOnExamples.kt | 83 +++++++++++++++++++++---- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index 3c5737a59..b8343f6a1 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -2,15 +2,13 @@ package prog8tests import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import kotlin.test.* +import kotlin.io.path.* + import prog8.compiler.compileProgram +import prog8.compiler.target.C64Target import prog8.compiler.target.Cx16Target import prog8.compiler.target.ICompilationTarget -import kotlin.io.path.Path -import kotlin.io.path.absolute -import kotlin.io.path.isDirectory -import kotlin.test.assertEquals -import kotlin.test.assertTrue - /** * ATTENTION: this is just kludge! @@ -23,8 +21,9 @@ class TestCompilerOnExamples { val examplesDir = workingDir.resolve("../examples") val outputDir = workingDir.resolve("build/tmp/test") + @Test - fun testDirectoriesSanityCheck() { + fun sanityCheckDirectories() { assertEquals("compiler", workingDir.fileName.toString()) assertTrue(examplesDir.isDirectory(), "sanity check; should be directory: $examplesDir") assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") @@ -49,30 +48,88 @@ class TestCompilerOnExamples { @Test - fun test_cxLogo_noopt() { + fun test_cxLogo_cx16_noopt() { testExample("cxlogo", Cx16Target, false) } @Test - fun test_cxLogo_opt() { + fun test_cxLogo_cx16_opt() { testExample("cxlogo", Cx16Target, true) } + @Test + fun test_cxLogo_c64_noopt() { + testExample("cxlogo", C64Target, false) + } + @Test + fun test_cxLogo_c64_opt() { + testExample("cxlogo", C64Target, true) + } @Test - fun test_swirl_noopt() { + fun test_swirl_cx16_noopt() { testExample("swirl", Cx16Target, false) } @Test - fun test_swirl_opt() { + fun test_swirl_cx16_opt() { testExample("swirl", Cx16Target, true) } + @Test + fun test_swirl_c64_noopt() { + testExample("swirl", C64Target, false) + } + @Test + fun test_swirl_c64_opt() { + testExample("swirl", C64Target, true) + } @Test - fun test_animals_noopt() { + fun test_animals_cx16_noopt() { testExample("animals", Cx16Target, false) } @Test - fun test_animals_opt() { + fun test_animals_cx16_opt() { testExample("animals", Cx16Target, true) } + @Test + fun test_animals_c64_noopt() { + testExample("animals", C64Target, false) + } + @Test + fun test_animals_c64_opt() { + testExample("animals", C64Target, true) + } + @Test + fun test_tehtriz_cx16_noopt() { + testExample("cx16/tehtriz", Cx16Target, false) + } + @Test + fun test_tehtriz_cx16_opt() { + testExample("cx16/tehtriz", Cx16Target, true) + } + @Test + fun test_tehtriz_c64_noopt() { + testExample("tehtriz", C64Target, false) + } + @Test + fun test_tehtriz_c64_opt() { + testExample("tehtriz", C64Target, true) + } + + // textelite.p8 is the largest example (~36KB) + @Test + fun test_textelite_cx16_noopt() { + testExample("textelite", Cx16Target, false) + } + @Test + fun test_textelite_cx16_opt() { + testExample("textelite", Cx16Target, true) + } + @Test + fun test_textelite_c64_noopt() { + testExample("textelite", C64Target, false) + } + @Test + fun test_textelite_c64_opt() { + testExample("textelite", C64Target, true) + } } From 0d73a7cd07c7bada5206ec0a4c4743cb99feba09 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 12:53:01 +0200 Subject: [PATCH 23/68] + add TestAstToSourceCode.kt (all 8 new tests failing due to missing semicolon) --- compilerAst/test/TestAstToSourceCode.kt | 139 ++++++++++++++++++++++++ compilerAst/test/TestProg8Parser.kt | 7 +- 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 compilerAst/test/TestAstToSourceCode.kt diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt new file mode 100644 index 000000000..8aaa8cd19 --- /dev/null +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -0,0 +1,139 @@ +package prog8tests + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.assertThrows +import kotlin.test.* + +import prog8.ast.* +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import prog8.parser.Prog8Parser.parseModule +import prog8.parser.SourceCode + +import prog8.ast.AstToSourceCode +import prog8.parser.ParseError + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestAstToSourceCode { + + object DummyFunctions: IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() + } + + object DummyMemsizer: IMemSizer { + override fun memorySize(dt: DataType): Int = 0 + } + + fun generateP8(module: Module) : String { + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) + module.linkParents(program) + module.program = program + + var generatedText = "" + val it = AstToSourceCode({ str -> generatedText += str }, program) + it.visit(program) + + return generatedText + } + + fun roundTrip(module: Module): Pair { + val generatedText = generateP8(module) + try { + val parsedAgain = parseModule(SourceCode.of(generatedText)) + return Pair(generatedText, parsedAgain) + } catch (e: ParseError) { + assert(false, { "should produce valid Prog8 but threw $e" }) + throw e + } + } + + @Test + fun testMentionsInternedStringsModule() { + val orig = SourceCode.of("\n") + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex(";.*$internedStringsModuleName")) + } + + @Test + fun testTargetDirectiveAndComment() { + val orig = SourceCode.of("%target 42 ; invalid arg - shouldn't matter\n") + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("%target +42")) + } + + @Test + fun testImportDirectiveWithLib() { + val orig = SourceCode.of("%import textio\n") + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("%import +textio")) + } + + @Test + fun testImportDirectiveWithUserModule() { + val orig = SourceCode.of("%import my_own_stuff\n") + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("%import +my_own_stuff")) + } + + + @Test + fun testStringLiteral_noAlt() { + val orig = SourceCode.of(""" + main { + str s = "fooBar\n" + } + """) + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("str +s += +\"fooBar\\\\n\"")) + } + + @Test + fun testStringLiteral_withAlt() { + val orig = SourceCode.of(""" + main { + str sAlt = @"fooBar\n" + } + """) + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("str +sAlt += +@\"fooBar\\\\n\"")) + } + + @Test + fun testCharLiteral_noAlt() { + val orig = SourceCode.of(""" + main { + ubyte c = 'x' + } + """) + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("ubyte +c += +'x'"), "char literal") + } + + @Test + fun testCharLiteral_withAlt() { + val orig = SourceCode.of(""" + main { + ubyte cAlt = @'x' + } + """) + val (txt, module) = roundTrip(parseModule(orig)) + // assertContains has *actual* first! + assertContains(txt, Regex("ubyte +cAlt += +@'x'"), "alt char literal") + } + +} diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 202464f5c..cb80c5030 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,15 +1,16 @@ package prog8tests import org.junit.jupiter.api.Test -import prog8.ast.Node -import prog8.ast.base.Position -import prog8.ast.statements.* import kotlin.test.* import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* + import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode +import prog8.ast.Node +import prog8.ast.base.Position +import prog8.ast.statements.* class TestProg8Parser { From 6c422216205a16a26192e665b93fc38bdaa58ffa Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 14:28:09 +0200 Subject: [PATCH 24/68] * fix AstToSourceCode: missing semicolon in header and footer, missing "@" for strings with altEncoding --- compilerAst/src/prog8/ast/AstToSourceCode.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/compilerAst/src/prog8/ast/AstToSourceCode.kt b/compilerAst/src/prog8/ast/AstToSourceCode.kt index 71fb3d7a5..9c42a7553 100644 --- a/compilerAst/src/prog8/ast/AstToSourceCode.kt +++ b/compilerAst/src/prog8/ast/AstToSourceCode.kt @@ -9,6 +9,11 @@ import prog8.ast.statements.* import prog8.ast.walk.IAstVisitor +/** + * Produces Prog8 source text from a [Program] (AST node), + * passing it as a String to the specified receiver function. + * TODO: rename/refactor to make proper sense in the presence of class [prog8.SourceCode] + */ class AstToSourceCode(val output: (text: String) -> Unit, val program: Program): IAstVisitor { private var scopelevel = 0 @@ -18,9 +23,9 @@ class AstToSourceCode(val output: (text: String) -> Unit, val program: Program): private fun outputi(s: Any) = output(indent(s.toString())) override fun visit(program: Program) { - outputln("============= PROGRAM ${program.name} (FROM AST) ===============") + outputln("; ============ PROGRAM ${program.name} (FROM AST) ==============") super.visit(program) - outputln("============= END PROGRAM ${program.name} (FROM AST) ===========") + outputln("; =========== END PROGRAM ${program.name} (FROM AST) ===========") } override fun visit(module: Module) { @@ -262,6 +267,8 @@ class AstToSourceCode(val output: (text: String) -> Unit, val program: Program): } override fun visit(string: StringLiteralValue) { + if (string.altEncoding) + output("@") output("\"${escape(string.value)}\"") } From cd295228ef3de6d52017346219cc1c586f3e42c3 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 15:19:20 +0200 Subject: [PATCH 25/68] + TestCompilerOnImportsAndIncludes.kt: 2 tests, both passing (but see FIXME in asmIncludeFromSameFolder.p8) --- .../compiler/target/cpu6502/codegen/AsmGen.kt | 3 + .../test/TestCompilerOnImportsAndIncludes.kt | 103 ++++++++++++++++++ .../test/fixtures/asmIncludeFromSameFolder.p8 | 13 +++ compiler/test/fixtures/foo_bar.asm | 2 + compiler/test/fixtures/foo_bar.p8 | 3 + .../test/fixtures/importFromSameFolder.p8 | 9 ++ 6 files changed, 133 insertions(+) create mode 100644 compiler/test/TestCompilerOnImportsAndIncludes.kt create mode 100644 compiler/test/fixtures/asmIncludeFromSameFolder.p8 create mode 100644 compiler/test/fixtures/foo_bar.asm create mode 100644 compiler/test/fixtures/foo_bar.p8 create mode 100644 compiler/test/fixtures/importFromSameFolder.p8 diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt index cc55baed0..95c3ae71a 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt @@ -1308,6 +1308,9 @@ $repeatLabel lda $counterVar } } + /** + * TODO: %asminclude and %asmbinary should be done earlier than code gen (-> put content into AST) + */ private fun translate(stmt: Directive) { when(stmt.directive) { "%asminclude" -> { diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt new file mode 100644 index 000000000..57d3cf960 --- /dev/null +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -0,0 +1,103 @@ +package prog8tests + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import kotlin.io.path.* +import kotlin.test.* + +import prog8.ast.expressions.AddressOf +import prog8.ast.expressions.IdentifierReference +import prog8.ast.expressions.StringLiteralValue +import prog8.ast.statements.FunctionCallStatement +import prog8.ast.statements.Label +import prog8.compiler.compileProgram +import prog8.compiler.target.Cx16Target + + +/** + * ATTENTION: this is just kludge! + * They are not really unit tests, but rather tests of the whole process, + * from source file loading all the way through to running 64tass. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestCompilerOnImportsAndIncludes { + val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! + val fixturesDir = workingDir.resolve("test/fixtures") + val outputDir = workingDir.resolve("build/tmp/test") + + @Test + fun sanityCheckDirectories() { + assertEquals("compiler", workingDir.fileName.toString()) + assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") + assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + } + + @Test + fun testImportFromSameFolder() { + val filepath = fixturesDir.resolve("importFromSameFolder.p8") + val imported = fixturesDir.resolve("foo_bar.p8") + + assertTrue(filepath.isRegularFile(), "sanity check; should be regular file: $filepath") + assertTrue(imported.isRegularFile(), "sanity check; should be regular file: $imported") + + val compilationTarget = Cx16Target + val result = compileProgram( + filepath, + optimize = false, + writeAssembly = true, + slowCodegenWarnings = false, + compilationTarget.name, + libdirs = listOf(), + outputDir + ) + assertTrue(result.success, "compilation should succeed") + + val program = result.programAst + val startSub = program.entrypoint() + val strLits = startSub.statements + .filterIsInstance() + .map { it.args[0] as IdentifierReference } + .map { it.targetVarDecl(program)!!.value as StringLiteralValue } + + assertEquals("main.bar", strLits[0].value) + assertEquals("foo.bar", strLits[1].value) + assertEquals("main", strLits[0].definingScope().name) + assertEquals("foo", strLits[1].definingScope().name) + } + + @Test + fun testAsmIncludeFromSameFolder() { + val filepath = fixturesDir.resolve("asmIncludeFromSameFolder.p8") + val included = fixturesDir.resolve("foo_bar.asm") + + assertTrue(filepath.isRegularFile(), "sanity check; should be regular file: $filepath") + assertTrue(included.isRegularFile(), "sanity check; should be regular file: $included") + + val compilationTarget = Cx16Target + val result = compileProgram( + filepath, + optimize = false, + writeAssembly = true, + slowCodegenWarnings = false, + compilationTarget.name, + libdirs = listOf(), + outputDir + ) + assertTrue(result.success, "compilation should succeed") + + val program = result.programAst + val startSub = program.entrypoint() + val args = startSub.statements + .filterIsInstance() + .map { it.args[0] } + + val str0 = (args[0] as IdentifierReference).targetVarDecl(program)!!.value as StringLiteralValue + assertEquals("main.bar", str0.value) + assertEquals("main", str0.definingScope().name) + + val id1 = (args[1] as AddressOf).identifier + val lbl1 = id1.targetStatement(program) as Label + assertEquals("foo_bar", lbl1.name) + assertEquals("start", lbl1.definingScope().name) + } +} diff --git a/compiler/test/fixtures/asmIncludeFromSameFolder.p8 b/compiler/test/fixtures/asmIncludeFromSameFolder.p8 new file mode 100644 index 000000000..df25f6e78 --- /dev/null +++ b/compiler/test/fixtures/asmIncludeFromSameFolder.p8 @@ -0,0 +1,13 @@ +%import textio +main { + str myBar = "main.bar" +;foo_bar: +; %asminclude "foo_bar.asm" ; FIXME: should be accessible from inside start() but give assembler error + sub start() { + txt.print(myBar) + txt.print(&foo_bar) + return +foo_bar: + %asminclude "foo_bar.asm" + } +} diff --git a/compiler/test/fixtures/foo_bar.asm b/compiler/test/fixtures/foo_bar.asm new file mode 100644 index 000000000..d415b6923 --- /dev/null +++ b/compiler/test/fixtures/foo_bar.asm @@ -0,0 +1,2 @@ +bar .text "foo.bar",0 + diff --git a/compiler/test/fixtures/foo_bar.p8 b/compiler/test/fixtures/foo_bar.p8 new file mode 100644 index 000000000..e60fa8199 --- /dev/null +++ b/compiler/test/fixtures/foo_bar.p8 @@ -0,0 +1,3 @@ +foo { + str bar = "foo.bar" +} diff --git a/compiler/test/fixtures/importFromSameFolder.p8 b/compiler/test/fixtures/importFromSameFolder.p8 new file mode 100644 index 000000000..3f233b10d --- /dev/null +++ b/compiler/test/fixtures/importFromSameFolder.p8 @@ -0,0 +1,9 @@ +%import textio +%import foo_bar +main { + str myBar = "main.bar" + sub start() { + txt.print(myBar) + txt.print(foo.bar) + } +} From 43c5ab8ecc61af4a4f9bcfed9c16ded66648f922 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 17:32:29 +0200 Subject: [PATCH 26/68] * refactor compilerAst tests, intro prog8test.helpers, @Disable the 3 tests that will pass after subsequent steps of "the plan" --- compilerAst/test/Helpers.kt | 77 ++++++++++++++++++ compilerAst/test/TestAstToSourceCode.kt | 25 ++---- compilerAst/test/TestModuleImporter.kt | 104 ++++++++++-------------- compilerAst/test/TestProg8Parser.kt | 43 ++++------ compilerAst/test/TestSourceCode.kt | 39 ++++----- 5 files changed, 151 insertions(+), 137 deletions(-) create mode 100644 compilerAst/test/Helpers.kt diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/Helpers.kt new file mode 100644 index 000000000..48748b43d --- /dev/null +++ b/compilerAst/test/Helpers.kt @@ -0,0 +1,77 @@ +package prog8tests.helpers + +import org.junit.jupiter.api.Test +import kotlin.test.* +import kotlin.io.path.* + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.IStringEncoding +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import java.nio.file.Path + + +val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! +val fixturesDir = workingDir.resolve("test/fixtures") +val resourcesDir = workingDir.resolve("res") +val outputDir = workingDir.resolve("build/tmp/test") + +inline fun assumeReadable(path: Path) { + assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") +} + +inline fun assumeReadableFile(path: Path) { + assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") +} + +inline fun assumeDirectory(path: Path) { + assertTrue(path.isDirectory(), "sanity check; should be directory: $path") +} + +inline fun assumeNotExists(path: Path) { + assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") +} + +inline fun sanityCheckDirectories(workingDirName: String? = null) { + if (workingDirName != null) + assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") + assumeDirectory(workingDir) + assumeDirectory(fixturesDir) + assumeDirectory(resourcesDir) + assumeDirectory(outputDir) + +} + + + + val DummyEncoding = object : IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("Not yet implemented") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("Not yet implemented") + } +} + +val DummyFunctions = object : IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue( + name: String, + args: List, + position: Position, + memsizer: IMemSizer + ): NumericLiteralValue? = null + + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() +} + +val DummyMemsizer = object : IMemSizer { + override fun memorySize(dt: DataType): Int = 0 +} + diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt index 8aaa8cd19..4f422a1b9 100644 --- a/compilerAst/test/TestAstToSourceCode.kt +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -1,16 +1,10 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.* import kotlin.test.* +import prog8tests.helpers.* import prog8.ast.* -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode @@ -21,17 +15,6 @@ import prog8.parser.ParseError @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestAstToSourceCode { - object DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - object DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } - fun generateP8(module: Module) : String { val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(program) @@ -50,7 +33,7 @@ class TestAstToSourceCode { val parsedAgain = parseModule(SourceCode.of(generatedText)) return Pair(generatedText, parsedAgain) } catch (e: ParseError) { - assert(false, { "should produce valid Prog8 but threw $e" }) + assert(false) { "should produce valid Prog8 but threw $e" } throw e } } @@ -113,6 +96,7 @@ class TestAstToSourceCode { } @Test + @Disabled fun testCharLiteral_noAlt() { val orig = SourceCode.of(""" main { @@ -125,6 +109,7 @@ class TestAstToSourceCode { } @Test + @Disabled fun testCharLiteral_withAlt() { val orig = SourceCode.of(""" main { diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 6fe9b6a73..b8458a2c3 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -3,18 +3,12 @@ package prog8tests import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import kotlin.test.* +import prog8tests.helpers.* + import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding import prog8.ast.Program -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import prog8.parser.ModuleImporter import prog8.parser.ParseError @@ -22,50 +16,29 @@ import prog8.parser.ParseError @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestModuleImporter { - object DummyEncoding: IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("Not yet implemented") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("Not yet implemented") - } - } - - object DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - object DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } - - @Test fun testImportModuleWithNonExistingPath() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) - val srcPath = Path.of("test", "fixtures", "i_do_not_exist") + val srcPath = fixturesDir.resolve("i_do_not_exist") + assumeNotExists(srcPath) - assertFalse(srcPath.exists(), "sanity check: file should not exist") assertFailsWith { importer.importModule(srcPath) } } @Test fun testImportModuleWithDirectoryPath() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) - val srcPath = Path.of("test", "fixtures") - - assertTrue(srcPath.isDirectory(), "sanity check: should be a directory") + val srcPath = fixturesDir + assumeDirectory(srcPath) // fn importModule(Path) used to check *.isReadable()*, but NOT .isRegularFile(): - assertTrue(srcPath.isReadable(), "sanity check: should still be readable") + assumeReadable(srcPath) assertFailsWith { importer.importModule(srcPath) } } @@ -73,10 +46,11 @@ class TestModuleImporter { @Test fun testImportModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) val filename = "file_with_syntax_error.p8" - val path = Path.of("test", "fixtures", filename) + val path = fixturesDir.resolve(filename) val act = { importer.importModule(path) } assertFailsWith { act() } @@ -93,14 +67,16 @@ class TestModuleImporter { @Test fun testImportModuleWithImportingModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) - val importing = Path.of("test", "fixtures", "import_file_with_syntax_error.p8") - val imported = Path.of("test", "fixtures", "file_with_syntax_error.p8") + val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") + val imported = fixturesDir.resolve("file_with_syntax_error.p8") + assumeReadableFile(importing) + assumeReadableFile(imported) val act = { importer.importModule(importing) } - assertTrue(importing.isRegularFile(), "sanity check: should be regular file") assertFailsWith { act() } try { act() @@ -116,14 +92,15 @@ class TestModuleImporter { @Test fun testImportLibraryModuleWithNonExistingName() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) val filenameNoExt = "i_do_not_exist" val filenameWithExt = filenameNoExt + ".p8" - val srcPathNoExt = Path.of("test", "fixtures", filenameNoExt) - val srcPathWithExt = Path.of("test", "fixtures", filenameWithExt) + val srcPathNoExt = fixturesDir.resolve(filenameNoExt) + val srcPathWithExt = fixturesDir.resolve(filenameWithExt) + assumeNotExists(srcPathNoExt) + assumeNotExists(srcPathWithExt) - assertFalse(srcPathNoExt.exists(), "sanity check: file should not exist") - assertFalse(srcPathWithExt.exists(), "sanity check: file should not exist") assertFailsWith { importer.importLibraryModule(filenameNoExt) } assertFailsWith { importer.importLibraryModule(filenameWithExt) } } @@ -131,18 +108,18 @@ class TestModuleImporter { @Test fun testImportLibraryModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf("./test/fixtures")) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val srcPath = fixturesDir.resolve("file_with_syntax_error.p8") + assumeReadableFile(srcPath) - val filename = "file_with_syntax_error" - - val act = { importer.importLibraryModule(filename) } + val act = { importer.importLibraryModule(srcPath.nameWithoutExtension) } assertFailsWith { act() } try { act() } catch (e: ParseError) { - val expectedProvenance = Path.of("test", "fixtures", filename + ".p8") - .absolutePathString() + val expectedProvenance = srcPath.absolutePathString() assertEquals(expectedProvenance, e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based") @@ -153,20 +130,21 @@ class TestModuleImporter { @Test fun testImportLibraryModuleWithImportingBadModule() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val libdirs = listOf("./test/fixtures") - val importer = ModuleImporter(program, DummyEncoding, "blah", libdirs) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) - val importing = "import_file_with_syntax_error" - val imported = "file_with_syntax_error" - val act = { importer.importLibraryModule(importing) } + val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") + val imported = fixturesDir.resolve("file_with_syntax_error.p8") + assumeReadableFile(importing) + assumeReadableFile(imported) + + val act = { importer.importLibraryModule(importing.nameWithoutExtension) } assertFailsWith { act() } try { act() } catch (e: ParseError) { - val expectedProvenance = Path.of(libdirs[0], imported + ".p8") - .normalize() - .absolutePathString() + val expectedProvenance = imported.normalize().absolutePathString() assertEquals(expectedProvenance, e.position.file) assertEquals(2, e.position.line, "line; should be 1-based") assertEquals(6, e.position.startCol, "startCol; should be 0-based") diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index cb80c5030..a718f98a0 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,8 +1,8 @@ package prog8tests -import org.junit.jupiter.api.Test +import org.junit.jupiter.api.* import kotlin.test.* -import java.nio.file.Path // TODO: use kotlin.io.path.Path instead +import prog8tests.helpers.* import kotlin.io.path.* import prog8.parser.ParseError @@ -13,14 +13,12 @@ import prog8.ast.base.Position import prog8.ast.statements.* +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestProg8Parser { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val fixturesDir = workingDir.resolve("test/fixtures") - @Test - fun testDirectoriesSanityCheck() { - assertEquals("compilerAst", workingDir.fileName.toString()) - assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compilerAst") } @Test @@ -164,15 +162,14 @@ class TestProg8Parser { @Test fun parseModuleShouldNotLookAtImports() { - val imported = "i_do_not_exist" - val pathNoExt = Path.of(imported).absolute() - val pathWithExt = Path.of("${pathNoExt}.p8") - val text = "%import $imported" - - assertFalse(pathNoExt.exists(), "sanity check: file should not exist: $pathNoExt") - assertFalse(pathWithExt.exists(), "sanity check: file should not exist: $pathWithExt") + val importedNoExt = fixturesDir.resolve("i_do_not_exist") + val importedWithExt = fixturesDir.resolve("i_do_not_exist.p8") + assumeNotExists(importedNoExt) + assumeNotExists(importedWithExt) + val text = "%import ${importedNoExt.name}" val module = parseModule(SourceCode.of(text)) + assertEquals(1, module.statements.size) } @@ -186,7 +183,7 @@ class TestProg8Parser { @Test fun testParseModuleWithEmptyFile() { val path = fixturesDir.resolve("empty.p8") - assertTrue(path.isRegularFile(), "sanity check: should be regular file: $path") + assumeReadableFile(path) val module = parseModule(SourceCode.fromPath(path)) assertEquals(0, module.statements.size) @@ -298,6 +295,7 @@ class TestProg8Parser { * TODO: this test is testing way too much at once */ @Test + @Disabled fun testInnerNodePositionsForSourceFromString() { val srcText = """ %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? @@ -335,17 +333,4 @@ class TestProg8Parser { assertPositionOf(whenStmt.choices[2], mpf, 9, 12) // TODO: endCol wrong! } - - @Test - fun testProg8Ast() { - val module = parseModule(SourceCode.of(""" - main { - sub start() { - return - } - } - """)) - assertIs(module.statements.first()) - assertEquals("main", (module.statements.first() as Block).name) - } } diff --git a/compilerAst/test/TestSourceCode.kt b/compilerAst/test/TestSourceCode.kt index 22e54cc12..dc9759fed 100644 --- a/compilerAst/test/TestSourceCode.kt +++ b/compilerAst/test/TestSourceCode.kt @@ -1,28 +1,20 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.* import kotlin.test.* +import prog8tests.helpers.* import kotlin.io.path.* import prog8.parser.SourceCode + @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestSourceCode { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val fixturesDir = workingDir.resolve("test/fixtures") - val resourcesDir = workingDir.resolve("res") - val outputDir = workingDir.resolve("build/tmp/test") - @Test - fun sanityCheckDirectories() { - assertEquals("compilerAst", workingDir.fileName.toString()) - assertTrue(workingDir.isDirectory(), "sanity check; should be directory: $workingDir") - assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") - assertTrue(resourcesDir.isDirectory(), "sanity check; should be directory: $resourcesDir") - assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compilerAst") } @Test @@ -41,8 +33,7 @@ class TestSourceCode { fun testFromPathWithNonExistingPath() { val filename = "i_do_not_exist.p8" val path = fixturesDir.resolve(filename) - - assertFalse(path.exists(), "sanity check: file should not exist: ${path.absolute()}") + assumeNotExists(path) assertFailsWith { SourceCode.fromPath(path) } } @@ -50,9 +41,7 @@ class TestSourceCode { fun testFromPathWithMissingExtension_p8() { val pathWithoutExt = fixturesDir.resolve("simple_main") val pathWithExt = Path(pathWithoutExt.toString() + ".p8") - - assertTrue(pathWithExt.isRegularFile(), "sanity check: should be normal file: ${pathWithExt.absolute()}") - assertTrue(pathWithExt.isReadable(), "sanity check: should be readable: ${pathWithExt.absolute()}") + assumeReadableFile(pathWithExt) assertFailsWith { SourceCode.fromPath(pathWithoutExt) } } @@ -108,7 +97,7 @@ class TestSourceCode { assertEquals("@embedded@$pathString", src.origin) - val expectedSrcText = resourcesDir.resolve(pathString.substringAfter("/")).toFile().readText() + val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() val actualSrcText = src.asString() assertEquals(expectedSrcText, actualSrcText) } @@ -133,7 +122,7 @@ class TestSourceCode { assertEquals("@embedded@$pathString", src.origin) - val expectedSrcText = resourcesDir.resolve(pathString.substringAfter("/")).toFile().readText() + val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() val actualSrcText = src.asString() assertEquals(expectedSrcText, actualSrcText) } @@ -145,7 +134,7 @@ class TestSourceCode { assertEquals("@embedded@/prog8lib/math.p8", src.origin) - val expectedSrcText = Path( "res", pathString).toFile().readText() + val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() val actualSrcText = src.asString() assertEquals(expectedSrcText, actualSrcText) assertTrue(src.isFromResources, ".isFromResources") @@ -155,15 +144,15 @@ class TestSourceCode { @Test fun testFromResourcesWithNonExistingFile_withLeadingSlash() { val pathString = "/prog8lib/i_do_not_exist" - val resPath = resourcesDir.resolve(pathString.substringAfter("/")) - assertFalse(resPath.exists(), "sanity check: should not exist: $resPath") + val resPath = resourcesDir.resolve(pathString.substring(1)) + assumeNotExists(resPath) assertThrows { SourceCode.fromResources(pathString) } } @Test fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() { val pathString = "prog8lib/i_do_not_exist" val resPath = resourcesDir.resolve(pathString) - assertFalse(resPath.exists(), "sanity check: should not exist: $resPath") + assumeNotExists(resPath) assertThrows { SourceCode.fromResources(pathString) } } From 5e194536a8efcdcd57c6029495eabe1b1a6f735e Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 18:18:27 +0200 Subject: [PATCH 27/68] * refactor compiler tests, again prog8test.helpers (TODO: remove duplication) --- compiler/test/AsmgenTests.kt | 20 +--- compiler/test/Helpers.kt | 101 ++++++++++++++++++ compiler/test/TestCompilerOnCharLit.kt | 69 +++--------- compiler/test/TestCompilerOnExamples.kt | 21 ++-- .../test/TestCompilerOnImportsAndIncludes.kt | 53 +++------ compiler/test/TestMemory.kt | 44 +++----- compiler/test/TestNumbers.kt | 7 +- compiler/test/TestNumericLiteralValue.kt | 11 +- compiler/test/TestPetscii.kt | 6 +- compiler/test/ZeropageTests.kt | 9 +- compilerAst/test/Helpers.kt | 1 - 11 files changed, 173 insertions(+), 169 deletions(-) create mode 100644 compiler/test/Helpers.kt diff --git a/compiler/test/AsmgenTests.kt b/compiler/test/AsmgenTests.kt index 20a77c71d..fe346680e 100644 --- a/compiler/test/AsmgenTests.kt +++ b/compiler/test/AsmgenTests.kt @@ -1,11 +1,10 @@ package prog8tests +import org.junit.jupiter.api.* import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer +import prog8tests.helpers.* + import prog8.ast.Module import prog8.ast.Program import prog8.ast.base.* @@ -17,18 +16,9 @@ import prog8.compiler.target.c64.C64MachineDefinition import prog8.compiler.target.cpu6502.codegen.AsmGen import java.nio.file.Path + @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestAsmGen6502 { - private class DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - private class DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } private fun createTestProgram(): Program { /* @@ -76,7 +66,7 @@ locallabel: val module = Module("test", mutableListOf(block), Position.DUMMY, null) module.linkParents(ParentSentinel) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.program = program return program } diff --git a/compiler/test/Helpers.kt b/compiler/test/Helpers.kt new file mode 100644 index 000000000..00d5f36ff --- /dev/null +++ b/compiler/test/Helpers.kt @@ -0,0 +1,101 @@ +package prog8tests.helpers + +import kotlin.test.* +import kotlin.io.path.* +import java.nio.file.Path + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.IStringEncoding +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import prog8.compiler.CompilationResult +import prog8.compiler.compileProgram +import prog8.compiler.target.ICompilationTarget + +// TODO: find a way to share with compiler/test/Helpers.kt, while still being able to amend it (-> compileFixture(..)) + +internal fun compileFixture( + filename: String, + platform: ICompilationTarget, + optimize: Boolean = false +) : CompilationResult { + val filepath = fixturesDir.resolve(filename) + assumeReadableFile(filepath) + val result = compileProgram( + filepath, + optimize, + writeAssembly = true, + slowCodegenWarnings = false, + platform.name, + libdirs = listOf(), + outputDir + ) + assertTrue(result.success, "compilation should succeed") + return result +} + +val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! +val fixturesDir = workingDir.resolve("test/fixtures") +val resourcesDir = workingDir.resolve("res") +val outputDir = workingDir.resolve("build/tmp/test") + +inline fun assumeReadable(path: Path) { + assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") +} + +inline fun assumeReadableFile(path: Path) { + assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") +} + +inline fun assumeDirectory(path: Path) { + assertTrue(path.isDirectory(), "sanity check; should be directory: $path") +} + +inline fun assumeNotExists(path: Path) { + assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") +} + +inline fun sanityCheckDirectories(workingDirName: String? = null) { + if (workingDirName != null) + assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") + assumeDirectory(workingDir) + assumeDirectory(fixturesDir) + assumeDirectory(resourcesDir) + assumeDirectory(outputDir) + +} + + +val DummyEncoding = object : IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + TODO("Not yet implemented") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + TODO("Not yet implemented") + } +} + +val DummyFunctions = object : IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue( + name: String, + args: List, + position: Position, + memsizer: IMemSizer + ): NumericLiteralValue? = null + + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() +} + +val DummyMemsizer = object : IMemSizer { + override fun memorySize(dt: DataType): Int = 0 +} + + + diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index 43344200e..01561bac2 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -1,16 +1,14 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import kotlin.io.path.* +import org.junit.jupiter.api.* import kotlin.test.* +import prog8tests.helpers.* import prog8.ast.IFunctionCall import prog8.ast.base.DataType import prog8.ast.base.VarDeclType import prog8.ast.expressions.IdentifierReference import prog8.ast.expressions.NumericLiteralValue -import prog8.compiler.compileProgram import prog8.compiler.target.Cx16Target @@ -21,31 +19,16 @@ import prog8.compiler.target.Cx16Target */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnCharLit { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val fixturesDir = workingDir.resolve("test/fixtures") - val outputDir = workingDir.resolve("build/tmp/test") - @Test - fun sanityCheckDirectories() { - assertEquals("compiler", workingDir.fileName.toString()) - assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") - assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") } @Test fun testCharLitAsRomsubArg() { - val filepath = fixturesDir.resolve("charLitAsRomsubArg.p8") - val compilationTarget = Cx16Target - val result = compileProgram( - filepath, - optimize = false, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "compilation should succeed") + val platform = Cx16Target + val result = compileFixture("charLitAsRomsubArg.p8", platform) val program = result.programAst val startSub = program.entrypoint() @@ -55,23 +38,14 @@ class TestCompilerOnCharLit { "char literal should have been replaced by ubyte literal") val arg = funCall.args[0] as NumericLiteralValue assertEquals(DataType.UBYTE, arg.type) - assertEquals(compilationTarget.encodeString("\n", false)[0], arg.number) + assertEquals(platform.encodeString("\n", false)[0], arg.number) } @Test fun testCharVarAsRomsubArg() { - val filepath = fixturesDir.resolve("charVarAsRomsubArg.p8") - val compilationTarget = Cx16Target - val result = compileProgram( - filepath, - optimize = false, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "compilation should succeed") + val platform = Cx16Target + val result = compileFixture("charVarAsRomsubArg.p8", platform) + val program = result.programAst val startSub = program.entrypoint() val funCall = startSub.statements.filterIsInstance()[0] @@ -91,23 +65,14 @@ class TestCompilerOnCharLit { "char literal should have been replaced by ubyte literal") val initializerValue = decl.value as NumericLiteralValue assertEquals(DataType.UBYTE, initializerValue.type) - assertEquals(compilationTarget.encodeString("\n", false)[0], initializerValue.number) + assertEquals(platform.encodeString("\n", false)[0], initializerValue.number) } @Test fun testCharConstAsRomsubArg() { - val filepath = fixturesDir.resolve("charConstAsRomsubArg.p8") - val compilationTarget = Cx16Target - val result = compileProgram( - filepath, - optimize = false, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "compilation should succeed") + val platform = Cx16Target + val result = compileFixture("charConstAsRomsubArg.p8", platform) + val program = result.programAst val startSub = program.entrypoint() val funCall = startSub.statements.filterIsInstance()[0] @@ -119,12 +84,12 @@ class TestCompilerOnCharLit { assertEquals(VarDeclType.CONST, decl.type) assertEquals(DataType.UBYTE, decl.datatype) assertEquals( - compilationTarget.encodeString("\n", false)[0], + platform.encodeString("\n", false)[0], (decl.value as NumericLiteralValue).number) } is NumericLiteralValue -> { assertEquals( - compilationTarget.encodeString("\n", false)[0], + platform.encodeString("\n", false)[0], arg.number) } else -> assertIs(funCall.args[0]) // make test fail diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index b8343f6a1..dec08e28a 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -1,9 +1,8 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.* import kotlin.test.* -import kotlin.io.path.* +import prog8tests.helpers.* import prog8.compiler.compileProgram import prog8.compiler.target.C64Target @@ -17,21 +16,17 @@ import prog8.compiler.target.ICompilationTarget */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnExamples { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val examplesDir = workingDir.resolve("../examples") - val outputDir = workingDir.resolve("build/tmp/test") + private val examplesDir = workingDir.resolve("../examples") - - @Test - fun sanityCheckDirectories() { - assertEquals("compiler", workingDir.fileName.toString()) - assertTrue(examplesDir.isDirectory(), "sanity check; should be directory: $examplesDir") - assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") + assumeDirectory(examplesDir) } // TODO: make assembly stage testable - in case of failure (eg of 64tass) it Process.exit s - fun testExample(nameWithoutExt: String, platform: ICompilationTarget, optimize: Boolean) { + private fun testExample(nameWithoutExt: String, platform: ICompilationTarget, optimize: Boolean) { val filepath = examplesDir.resolve("$nameWithoutExt.p8") val result = compileProgram( filepath, diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index 57d3cf960..835c1ea1f 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -1,9 +1,8 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance -import kotlin.io.path.* +import org.junit.jupiter.api.* import kotlin.test.* +import prog8tests.helpers.* import prog8.ast.expressions.AddressOf import prog8.ast.expressions.IdentifierReference @@ -12,6 +11,7 @@ import prog8.ast.statements.FunctionCallStatement import prog8.ast.statements.Label import prog8.compiler.compileProgram import prog8.compiler.target.Cx16Target +import kotlin.io.path.name /** @@ -21,36 +21,21 @@ import prog8.compiler.target.Cx16Target */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnImportsAndIncludes { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val fixturesDir = workingDir.resolve("test/fixtures") - val outputDir = workingDir.resolve("build/tmp/test") - @Test - fun sanityCheckDirectories() { - assertEquals("compiler", workingDir.fileName.toString()) - assertTrue(fixturesDir.isDirectory(), "sanity check; should be directory: $fixturesDir") - assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") } @Test fun testImportFromSameFolder() { val filepath = fixturesDir.resolve("importFromSameFolder.p8") val imported = fixturesDir.resolve("foo_bar.p8") + assumeReadableFile(filepath) + assumeReadableFile(imported) - assertTrue(filepath.isRegularFile(), "sanity check; should be regular file: $filepath") - assertTrue(imported.isRegularFile(), "sanity check; should be regular file: $imported") - - val compilationTarget = Cx16Target - val result = compileProgram( - filepath, - optimize = false, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "compilation should succeed") + val platform = Cx16Target + val result = compileFixture(filepath.name, platform) val program = result.programAst val startSub = program.entrypoint() @@ -69,21 +54,11 @@ class TestCompilerOnImportsAndIncludes { fun testAsmIncludeFromSameFolder() { val filepath = fixturesDir.resolve("asmIncludeFromSameFolder.p8") val included = fixturesDir.resolve("foo_bar.asm") + assumeReadableFile(filepath) + assumeReadableFile(included) - assertTrue(filepath.isRegularFile(), "sanity check; should be regular file: $filepath") - assertTrue(included.isRegularFile(), "sanity check; should be regular file: $included") - - val compilationTarget = Cx16Target - val result = compileProgram( - filepath, - optimize = false, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "compilation should succeed") + val platform = Cx16Target + val result = compileFixture(filepath.name, platform) val program = result.programAst val startSub = program.entrypoint() diff --git a/compiler/test/TestMemory.kt b/compiler/test/TestMemory.kt index d0e926aa6..4256398ab 100644 --- a/compiler/test/TestMemory.kt +++ b/compiler/test/TestMemory.kt @@ -1,8 +1,9 @@ package prog8tests -import org.junit.jupiter.api.Test -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer +import org.junit.jupiter.api.* +import kotlin.test.* +import prog8tests.helpers.* + import prog8.ast.Module import prog8.ast.Program import prog8.ast.base.DataType @@ -12,29 +13,16 @@ import prog8.ast.base.VarDeclType import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.compiler.target.C64Target -import java.nio.file.Path -import kotlin.test.assertFalse -import kotlin.test.assertTrue - +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestMemory { - private class DummyFunctions: IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? = null - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() - } - - private class DummyMemsizer: IMemSizer { - override fun memorySize(dt: DataType): Int = 0 - } - + @Test fun testInValidRamC64_memory_addresses() { var memexpr = NumericLiteralValue.optimalInteger(0x0000, Position.DUMMY) var target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) assertTrue(C64Target.isInRegularRAM(target, program)) memexpr = NumericLiteralValue.optimalInteger(0x1000, Position.DUMMY) @@ -59,7 +47,7 @@ class TestMemory { var memexpr = NumericLiteralValue.optimalInteger(0xa000, Position.DUMMY) var target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) assertFalse(C64Target.isInRegularRAM(target, program)) memexpr = NumericLiteralValue.optimalInteger(0xafff, Position.DUMMY) @@ -78,7 +66,7 @@ class TestMemory { @Test fun testInValidRamC64_memory_identifiers() { var target = createTestProgramForMemoryRefViaVar(0x1000, VarDeclType.VAR) - val program = Program("test", mutableListOf(), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) assertTrue(C64Target.isInRegularRAM(target, program)) target = createTestProgramForMemoryRefViaVar(0xd020, VarDeclType.VAR) @@ -107,7 +95,7 @@ class TestMemory { fun testInValidRamC64_memory_expression() { val memexpr = PrefixExpression("+", NumericLiteralValue.optimalInteger(0x1000, Position.DUMMY), Position.DUMMY) val target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) assertFalse(C64Target.isInRegularRAM(target, program)) } @@ -118,7 +106,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -131,7 +119,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -144,7 +132,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertFalse(C64Target.isInRegularRAM(target, program)) } @@ -157,7 +145,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -171,7 +159,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -185,7 +173,7 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions(), DummyMemsizer()) + val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(ParentSentinel) assertFalse(C64Target.isInRegularRAM(target, program)) } diff --git a/compiler/test/TestNumbers.kt b/compiler/test/TestNumbers.kt index 1de09e330..499db45d7 100644 --- a/compiler/test/TestNumbers.kt +++ b/compiler/test/TestNumbers.kt @@ -1,17 +1,16 @@ package prog8tests +import org.junit.jupiter.api.* +import kotlin.test.* import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.closeTo import org.hamcrest.Matchers.equalTo -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance + import prog8.ast.toHex import prog8.compiler.CompilerException import prog8.compiler.target.c64.C64MachineDefinition.FLOAT_MAX_NEGATIVE import prog8.compiler.target.c64.C64MachineDefinition.FLOAT_MAX_POSITIVE import prog8.compiler.target.c64.C64MachineDefinition.Mflpt5 -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestNumbers { diff --git a/compiler/test/TestNumericLiteralValue.kt b/compiler/test/TestNumericLiteralValue.kt index 00666de65..22d9ae03e 100644 --- a/compiler/test/TestNumericLiteralValue.kt +++ b/compiler/test/TestNumericLiteralValue.kt @@ -1,19 +1,14 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.* +import kotlin.test.* + import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.ArrayLiteralValue import prog8.ast.expressions.InferredTypes import prog8.ast.expressions.NumericLiteralValue import prog8.ast.expressions.StringLiteralValue -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertNotEquals -import kotlin.test.assertTrue - - @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestNumericLiteralValue { diff --git a/compiler/test/TestPetscii.kt b/compiler/test/TestPetscii.kt index c83657810..0fcc9a2dc 100644 --- a/compiler/test/TestPetscii.kt +++ b/compiler/test/TestPetscii.kt @@ -1,15 +1,15 @@ package prog8tests +import org.junit.jupiter.api.* +import kotlin.test.* import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance + import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.NumericLiteralValue import prog8.ast.expressions.StringLiteralValue import prog8.compiler.target.cbm.Petscii -import kotlin.test.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) diff --git a/compiler/test/ZeropageTests.kt b/compiler/test/ZeropageTests.kt index 8e2a3a6b7..4ece96e5e 100644 --- a/compiler/test/ZeropageTests.kt +++ b/compiler/test/ZeropageTests.kt @@ -1,17 +1,14 @@ package prog8tests -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.* +import kotlin.test.* + import prog8.ast.base.DataType import prog8.compiler.* import prog8.compiler.target.C64Target import prog8.compiler.target.Cx16Target import prog8.compiler.target.c64.C64MachineDefinition.C64Zeropage import prog8.compiler.target.cx16.CX16MachineDefinition.CX16Zeropage -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertFalse -import kotlin.test.assertTrue @TestInstance(TestInstance.Lifecycle.PER_CLASS) diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/Helpers.kt index 48748b43d..0ffc8a5a4 100644 --- a/compilerAst/test/Helpers.kt +++ b/compilerAst/test/Helpers.kt @@ -47,7 +47,6 @@ inline fun sanityCheckDirectories(workingDirName: String? = null) { } - val DummyEncoding = object : IStringEncoding { override fun encodeString(str: String, altEncoding: Boolean): List { TODO("Not yet implemented") From c80a15846d627f5080b5d9a8e3c71d31b6e32871 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 11 Jul 2021 19:04:53 +0200 Subject: [PATCH 28/68] * some more housekeeping re tests: gradle doesn't like .* imports for annotations, added @Disabled comments, made warnings go away --- compiler/test/AsmgenTests.kt | 3 +- compiler/test/Helpers.kt | 10 +++---- compiler/test/TestCompilerOnCharLit.kt | 4 ++- compiler/test/TestCompilerOnExamples.kt | 6 +++- .../test/TestCompilerOnImportsAndIncludes.kt | 7 +++-- compiler/test/TestMemory.kt | 4 ++- compiler/test/TestNumbers.kt | 3 +- compiler/test/TestNumericLiteralValue.kt | 3 +- compiler/test/TestPetscii.kt | 3 +- compiler/test/ZeropageTests.kt | 3 +- compilerAst/test/Helpers.kt | 10 +++---- compilerAst/test/TestAstToSourceCode.kt | 28 ++++++++++--------- compilerAst/test/TestModuleImporter.kt | 3 +- compilerAst/test/TestProg8Parser.kt | 7 +++-- compilerAst/test/TestSourceCode.kt | 10 +++---- 15 files changed, 61 insertions(+), 43 deletions(-) diff --git a/compiler/test/AsmgenTests.kt b/compiler/test/AsmgenTests.kt index fe346680e..34ecf58cb 100644 --- a/compiler/test/AsmgenTests.kt +++ b/compiler/test/AsmgenTests.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo import prog8tests.helpers.* diff --git a/compiler/test/Helpers.kt b/compiler/test/Helpers.kt index 00d5f36ff..de5a55217 100644 --- a/compiler/test/Helpers.kt +++ b/compiler/test/Helpers.kt @@ -43,23 +43,23 @@ val fixturesDir = workingDir.resolve("test/fixtures") val resourcesDir = workingDir.resolve("res") val outputDir = workingDir.resolve("build/tmp/test") -inline fun assumeReadable(path: Path) { +fun assumeReadable(path: Path) { assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") } -inline fun assumeReadableFile(path: Path) { +fun assumeReadableFile(path: Path) { assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") } -inline fun assumeDirectory(path: Path) { +fun assumeDirectory(path: Path) { assertTrue(path.isDirectory(), "sanity check; should be directory: $path") } -inline fun assumeNotExists(path: Path) { +fun assumeNotExists(path: Path) { assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") } -inline fun sanityCheckDirectories(workingDirName: String? = null) { +fun sanityCheckDirectories(workingDirName: String? = null) { if (workingDirName != null) assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") assumeDirectory(workingDir) diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index 01561bac2..2c61ea191 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -1,6 +1,8 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll import kotlin.test.* import prog8tests.helpers.* diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index dec08e28a..56932460a 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -1,6 +1,9 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Disabled import kotlin.test.* import prog8tests.helpers.* @@ -14,6 +17,7 @@ import prog8.compiler.target.ICompilationTarget * They are not really unit tests, but rather tests of the whole process, * from source file loading all the way through to running 64tass. */ +//@Disabled("to save some time") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnExamples { private val examplesDir = workingDir.resolve("../examples") diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index 835c1ea1f..4ba084a39 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -1,7 +1,10 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll import kotlin.test.* +import kotlin.io.path.* import prog8tests.helpers.* import prog8.ast.expressions.AddressOf @@ -9,9 +12,7 @@ import prog8.ast.expressions.IdentifierReference import prog8.ast.expressions.StringLiteralValue import prog8.ast.statements.FunctionCallStatement import prog8.ast.statements.Label -import prog8.compiler.compileProgram import prog8.compiler.target.Cx16Target -import kotlin.io.path.name /** diff --git a/compiler/test/TestMemory.kt b/compiler/test/TestMemory.kt index 4256398ab..25a1bf3bb 100644 --- a/compiler/test/TestMemory.kt +++ b/compiler/test/TestMemory.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import prog8tests.helpers.* @@ -14,6 +15,7 @@ import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.compiler.target.C64Target + @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestMemory { diff --git a/compiler/test/TestNumbers.kt b/compiler/test/TestNumbers.kt index 499db45d7..a22d98e68 100644 --- a/compiler/test/TestNumbers.kt +++ b/compiler/test/TestNumbers.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.closeTo diff --git a/compiler/test/TestNumericLiteralValue.kt b/compiler/test/TestNumericLiteralValue.kt index 22d9ae03e..5492fc2cf 100644 --- a/compiler/test/TestNumericLiteralValue.kt +++ b/compiler/test/TestNumericLiteralValue.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import prog8.ast.base.DataType diff --git a/compiler/test/TestPetscii.kt b/compiler/test/TestPetscii.kt index 0fcc9a2dc..f658e93c0 100644 --- a/compiler/test/TestPetscii.kt +++ b/compiler/test/TestPetscii.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo diff --git a/compiler/test/ZeropageTests.kt b/compiler/test/ZeropageTests.kt index 4ece96e5e..01622d02a 100644 --- a/compiler/test/ZeropageTests.kt +++ b/compiler/test/ZeropageTests.kt @@ -1,6 +1,7 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import prog8.ast.base.DataType diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/Helpers.kt index 0ffc8a5a4..5fc3d8870 100644 --- a/compilerAst/test/Helpers.kt +++ b/compilerAst/test/Helpers.kt @@ -20,23 +20,23 @@ val fixturesDir = workingDir.resolve("test/fixtures") val resourcesDir = workingDir.resolve("res") val outputDir = workingDir.resolve("build/tmp/test") -inline fun assumeReadable(path: Path) { +fun assumeReadable(path: Path) { assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") } -inline fun assumeReadableFile(path: Path) { +fun assumeReadableFile(path: Path) { assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") } -inline fun assumeDirectory(path: Path) { +fun assumeDirectory(path: Path) { assertTrue(path.isDirectory(), "sanity check; should be directory: $path") } -inline fun assumeNotExists(path: Path) { +fun assumeNotExists(path: Path) { assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") } -inline fun sanityCheckDirectories(workingDirName: String? = null) { +fun sanityCheckDirectories(workingDirName: String? = null) { if (workingDirName != null) assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") assumeDirectory(workingDir) diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt index 4f422a1b9..563b58528 100644 --- a/compilerAst/test/TestAstToSourceCode.kt +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -1,6 +1,8 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Disabled import kotlin.test.* import prog8tests.helpers.* @@ -15,7 +17,7 @@ import prog8.parser.ParseError @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestAstToSourceCode { - fun generateP8(module: Module) : String { + private fun generateP8(module: Module) : String { val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) module.linkParents(program) module.program = program @@ -27,7 +29,7 @@ class TestAstToSourceCode { return generatedText } - fun roundTrip(module: Module): Pair { + private fun roundTrip(module: Module): Pair { val generatedText = generateP8(module) try { val parsedAgain = parseModule(SourceCode.of(generatedText)) @@ -41,7 +43,7 @@ class TestAstToSourceCode { @Test fun testMentionsInternedStringsModule() { val orig = SourceCode.of("\n") - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex(";.*$internedStringsModuleName")) } @@ -49,7 +51,7 @@ class TestAstToSourceCode { @Test fun testTargetDirectiveAndComment() { val orig = SourceCode.of("%target 42 ; invalid arg - shouldn't matter\n") - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("%target +42")) } @@ -57,7 +59,7 @@ class TestAstToSourceCode { @Test fun testImportDirectiveWithLib() { val orig = SourceCode.of("%import textio\n") - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("%import +textio")) } @@ -65,7 +67,7 @@ class TestAstToSourceCode { @Test fun testImportDirectiveWithUserModule() { val orig = SourceCode.of("%import my_own_stuff\n") - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("%import +my_own_stuff")) } @@ -78,7 +80,7 @@ class TestAstToSourceCode { str s = "fooBar\n" } """) - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("str +s += +\"fooBar\\\\n\"")) } @@ -90,33 +92,33 @@ class TestAstToSourceCode { str sAlt = @"fooBar\n" } """) - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("str +sAlt += +@\"fooBar\\\\n\"")) } @Test - @Disabled + @Disabled("TODO: char literals should be kept until code gen - step 4, 'introduce AST node CharLiteral'") fun testCharLiteral_noAlt() { val orig = SourceCode.of(""" main { ubyte c = 'x' } """) - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("ubyte +c += +'x'"), "char literal") } @Test - @Disabled + @Disabled("TODO: char literals should be kept until code gen - step 4, 'introduce AST node CharLiteral'") fun testCharLiteral_withAlt() { val orig = SourceCode.of(""" main { ubyte cAlt = @'x' } """) - val (txt, module) = roundTrip(parseModule(orig)) + val (txt, _) = roundTrip(parseModule(orig)) // assertContains has *actual* first! assertContains(txt, Regex("ubyte +cAlt += +@'x'"), "alt char literal") } diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index b8458a2c3..68ebb41e4 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -1,11 +1,10 @@ package prog8tests -import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test import kotlin.test.* import prog8tests.helpers.* -import java.nio.file.Path // TODO: use kotlin.io.path.Path instead import kotlin.io.path.* import prog8.ast.Program diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index a718f98a0..42c8d67c4 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,6 +1,9 @@ package prog8tests -import org.junit.jupiter.api.* +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.BeforeAll import kotlin.test.* import prog8tests.helpers.* import kotlin.io.path.* @@ -295,7 +298,7 @@ class TestProg8Parser { * TODO: this test is testing way too much at once */ @Test - @Disabled + @Disabled("TODO: fix .position of nodes below Module - step 8, 'refactor AST gen'") fun testInnerNodePositionsForSourceFromString() { val srcText = """ %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? diff --git a/compilerAst/test/TestSourceCode.kt b/compilerAst/test/TestSourceCode.kt index dc9759fed..571f23eb2 100644 --- a/compilerAst/test/TestSourceCode.kt +++ b/compilerAst/test/TestSourceCode.kt @@ -1,5 +1,8 @@ package prog8tests +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.* import kotlin.test.* import prog8tests.helpers.* @@ -8,7 +11,6 @@ import kotlin.io.path.* import prog8.parser.SourceCode - @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestSourceCode { @@ -156,10 +158,8 @@ class TestSourceCode { assertThrows { SourceCode.fromResources(pathString) } } - /** - * TODO("inside resources: cannot tell apart a folder from a file") - */ - //@Test + @Test + @Disabled("TODO: inside resources: cannot tell apart a folder from a file") fun testFromResourcesWithDirectory() { val pathString = "/prog8lib" assertThrows { SourceCode.fromResources(pathString) } From 0567168ea98214ca23faa1d510c4117f6e806df5 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 21 Jun 2021 20:16:50 +0200 Subject: [PATCH 29/68] + add AST node CharLiteral, *without* turning them into ubyte s. This breaks tests, particularly 3 in TestCompilerOnCharLit. I'm comitting this separately since the failure modes might be of interest (compiler says "internal error"). --- .../compiler/astprocessing/AstChecker.kt | 12 ++- compiler/test/Helpers.kt | 4 +- compilerAst/src/prog8/ast/AstToSourceCode.kt | 6 ++ .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 13 +-- .../prog8/ast/expressions/AstExpressions.kt | 31 ++++++ compilerAst/src/prog8/ast/walk/AstWalker.kt | 7 ++ compilerAst/src/prog8/ast/walk/IAstVisitor.kt | 3 + compilerAst/test/Helpers.kt | 3 +- compilerAst/test/TestAstToSourceCode.kt | 2 - compilerAst/test/TestProg8Parser.kt | 99 ++++++++++++++++++- 10 files changed, 161 insertions(+), 19 deletions(-) diff --git a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt index 20c9a830f..296a4ca01 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstChecker.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstChecker.kt @@ -776,10 +776,20 @@ internal class AstChecker(private val program: Program, super.visit(array) } + override fun visit(char: CharLiteral) { + try { // just *try* if it can be encoded, don't actually do it + compTarget.encodeString(char.value.toString(), char.altEncoding) + } catch (cx: CharConversionException) { + errors.err(cx.message ?: "can't encode character", char.position) + } + + super.visit(char) + } + override fun visit(string: StringLiteralValue) { checkValueTypeAndRangeString(DataType.STR, string) - try { + try { // just *try* if it can be encoded, don't actually do it compTarget.encodeString(string.value, string.altEncoding) } catch (cx: CharConversionException) { errors.err(cx.message ?: "can't encode string", string.position) diff --git a/compiler/test/Helpers.kt b/compiler/test/Helpers.kt index de5a55217..cecef98af 100644 --- a/compiler/test/Helpers.kt +++ b/compiler/test/Helpers.kt @@ -48,6 +48,7 @@ fun assumeReadable(path: Path) { } fun assumeReadableFile(path: Path) { + assumeReadable(path) assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") } @@ -66,7 +67,6 @@ fun sanityCheckDirectories(workingDirName: String? = null) { assumeDirectory(fixturesDir) assumeDirectory(resourcesDir) assumeDirectory(outputDir) - } @@ -97,5 +97,3 @@ val DummyMemsizer = object : IMemSizer { override fun memorySize(dt: DataType): Int = 0 } - - diff --git a/compilerAst/src/prog8/ast/AstToSourceCode.kt b/compilerAst/src/prog8/ast/AstToSourceCode.kt index 9c42a7553..6896f85ad 100644 --- a/compilerAst/src/prog8/ast/AstToSourceCode.kt +++ b/compilerAst/src/prog8/ast/AstToSourceCode.kt @@ -266,6 +266,12 @@ class AstToSourceCode(val output: (text: String) -> Unit, val program: Program): output(numLiteral.number.toString()) } + override fun visit(char: CharLiteral) { + if (char.altEncoding) + output("@") + output("'${escape(char.value.toString())}'") + } + override fun visit(string: StringLiteralValue) { if (string.altEncoding) output("@") diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index 1792215b2..c2c81dd1c 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -435,15 +435,7 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) } litval.floatliteral()!=null -> NumericLiteralValue(DataType.FLOAT, litval.floatliteral().toAst(), litval.toPosition()) litval.stringliteral()!=null -> litval.stringliteral().toAst() - litval.charliteral()!=null -> { - try { - NumericLiteralValue(DataType.UBYTE, encoding.encodeString( - unescape(litval.charliteral().SINGLECHAR().text, litval.toPosition()), - litval.charliteral().ALT_STRING_ENCODING()!=null)[0], litval.toPosition()) - } catch (ce: CharConversionException) { - throw SyntaxError(ce.message ?: ce.toString(), litval.toPosition()) - } - } + litval.charliteral()!=null -> litval.charliteral().toAst() litval.arrayliteral()!=null -> { val array = litval.arrayliteral().toAst(encoding) // the actual type of the arraysize can not yet be determined here (missing namespace & heap) @@ -491,6 +483,9 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) throw FatalAstException(text) } +private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral = + CharLiteral(unescape(this.SINGLECHAR().text, toPosition())[0], this.ALT_STRING_ENCODING() != null, toPosition()) + private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue = StringLiteralValue(unescape(this.STRING().text, toPosition()), ALT_STRING_ENCODING()!=null, toPosition()) diff --git a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt index e4ea9097a..d9aa47362 100644 --- a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt +++ b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt @@ -499,6 +499,37 @@ class NumericLiteralValue(val type: DataType, // only numerical types allowed } } +class CharLiteral(val value: Char, + val altEncoding: Boolean, // such as: screencodes instead of Petscii for the C64 + override val position: Position) : Expression() { + override lateinit var parent: Node + + override fun linkParents(parent: Node) { + this.parent = parent + } + + override val isSimple = true + + override fun replaceChildNode(node: Node, replacement: Node) { + throw FatalAstException("can't replace here") + } + + override fun referencesIdentifier(vararg scopedName: String) = false + override fun constValue(program: Program): NumericLiteralValue? = null // TODO: CharLiteral.constValue can't be NumericLiteralValue... + override fun accept(visitor: IAstVisitor) = visitor.visit(this) + override fun accept(walker: AstWalker, parent: Node) = walker.visit(this, parent) + + override fun toString(): String = "'${escape(value.toString())}'" + override fun inferType(program: Program): InferredTypes.InferredType = InferredTypes.knownFor(DataType.UNDEFINED) // FIXME: CharLiteral.inferType + operator fun compareTo(other: CharLiteral): Int = value.compareTo(other.value) + override fun hashCode(): Int = Objects.hash(value, altEncoding) + override fun equals(other: Any?): Boolean { + if (other == null || other !is CharLiteral) + return false + return value == other.value && altEncoding == other.altEncoding + } +} + class StringLiteralValue(val value: String, val altEncoding: Boolean, // such as: screencodes instead of Petscii for the C64 override val position: Position) : Expression() { diff --git a/compilerAst/src/prog8/ast/walk/AstWalker.kt b/compilerAst/src/prog8/ast/walk/AstWalker.kt index 194b1f2e6..91a10db12 100644 --- a/compilerAst/src/prog8/ast/walk/AstWalker.kt +++ b/compilerAst/src/prog8/ast/walk/AstWalker.kt @@ -110,6 +110,7 @@ abstract class AstWalker { open fun before(untilLoop: UntilLoop, parent: Node): Iterable = noModifications open fun before(returnStmt: Return, parent: Node): Iterable = noModifications open fun before(scope: AnonymousScope, parent: Node): Iterable = noModifications + open fun before(char: CharLiteral, parent: Node): Iterable = noModifications open fun before(string: StringLiteralValue, parent: Node): Iterable = noModifications open fun before(subroutine: Subroutine, parent: Node): Iterable = noModifications open fun before(typecast: TypecastExpression, parent: Node): Iterable = noModifications @@ -150,6 +151,7 @@ abstract class AstWalker { open fun after(untilLoop: UntilLoop, parent: Node): Iterable = noModifications open fun after(returnStmt: Return, parent: Node): Iterable = noModifications open fun after(scope: AnonymousScope, parent: Node): Iterable = noModifications + open fun after(char: CharLiteral, parent: Node): Iterable = noModifications open fun after(string: StringLiteralValue, parent: Node): Iterable = noModifications open fun after(subroutine: Subroutine, parent: Node): Iterable = noModifications open fun after(typecast: TypecastExpression, parent: Node): Iterable = noModifications @@ -300,6 +302,11 @@ abstract class AstWalker { track(after(numLiteral, parent), numLiteral, parent) } + fun visit(char: CharLiteral, parent: Node) { + track(before(char, parent), char, parent) + track(after(char, parent), char, parent) + } + fun visit(string: StringLiteralValue, parent: Node) { track(before(string, parent), string, parent) track(after(string, parent), string, parent) diff --git a/compilerAst/src/prog8/ast/walk/IAstVisitor.kt b/compilerAst/src/prog8/ast/walk/IAstVisitor.kt index a2fcdc673..ca71f94bb 100644 --- a/compilerAst/src/prog8/ast/walk/IAstVisitor.kt +++ b/compilerAst/src/prog8/ast/walk/IAstVisitor.kt @@ -79,6 +79,9 @@ interface IAstVisitor { fun visit(numLiteral: NumericLiteralValue) { } + fun visit(char: CharLiteral) { + } + fun visit(string: StringLiteralValue) { } diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/Helpers.kt index 5fc3d8870..a49aed04a 100644 --- a/compilerAst/test/Helpers.kt +++ b/compilerAst/test/Helpers.kt @@ -1,6 +1,5 @@ package prog8tests.helpers -import org.junit.jupiter.api.Test import kotlin.test.* import kotlin.io.path.* @@ -25,6 +24,7 @@ fun assumeReadable(path: Path) { } fun assumeReadableFile(path: Path) { + assumeReadable(path) assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") } @@ -43,7 +43,6 @@ fun sanityCheckDirectories(workingDirName: String? = null) { assumeDirectory(fixturesDir) assumeDirectory(resourcesDir) assumeDirectory(outputDir) - } diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt index 563b58528..7b6166075 100644 --- a/compilerAst/test/TestAstToSourceCode.kt +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -98,7 +98,6 @@ class TestAstToSourceCode { } @Test - @Disabled("TODO: char literals should be kept until code gen - step 4, 'introduce AST node CharLiteral'") fun testCharLiteral_noAlt() { val orig = SourceCode.of(""" main { @@ -111,7 +110,6 @@ class TestAstToSourceCode { } @Test - @Disabled("TODO: char literals should be kept until code gen - step 4, 'introduce AST node CharLiteral'") fun testCharLiteral_withAlt() { val orig = SourceCode.of(""" main { diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 42c8d67c4..9826bf908 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -11,9 +11,10 @@ import kotlin.io.path.* import prog8.parser.ParseError import prog8.parser.Prog8Parser.parseModule import prog8.parser.SourceCode -import prog8.ast.Node -import prog8.ast.base.Position +import prog8.ast.* import prog8.ast.statements.* +import prog8.ast.base.Position +import prog8.ast.expressions.CharLiteral @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -336,4 +337,98 @@ class TestProg8Parser { assertPositionOf(whenStmt.choices[2], mpf, 9, 12) // TODO: endCol wrong! } + @Test + fun testCharLitAsArg() { + val src = SourceCode.of(""" + main { + sub start() { + chrout('\n') + } + } + """) + val module = parseModule(src) + + val startSub = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + val funCall = startSub.statements.filterIsInstance().first() + + assertIs(funCall.args[0]) + val char = funCall.args[0] as CharLiteral + assertEquals('\n', char.value) + } + + @Test + fun testBlockLevelVarDeclWithCharLiteral_noAltEnc() { + val src = SourceCode.of(""" + main { + ubyte c = 'x' + } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") + } + + @Test + fun testBlockLevelConstDeclWithCharLiteral_withAltEnc() { + val src = SourceCode.of(""" + main { + const ubyte c = @'x' + } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") + } + + @Test + fun testSubRoutineLevelVarDeclWithCharLiteral_noAltEnc() { + val src = SourceCode.of(""" + main { + sub start() { + ubyte c = 'x' + } + } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") + } + + @Test + fun testSubRoutineLevelConstDeclWithCharLiteral_withAltEnc() { + val src = SourceCode.of(""" + main { + sub start() { + const ubyte c = @'x' + } + } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") + } } From 82f5a141ed580b6e7a79f6c255c13dd4fc77b6d5 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 28 Jun 2021 12:04:34 +0200 Subject: [PATCH 30/68] * reintroduce the conversion of CharLiteral to UBYTE literals, but now *during AST preprocessing*, not in the parser --- compiler/src/prog8/compiler/Compiler.kt | 8 +++---- .../compiler/astprocessing/AstExtensions.kt | 21 +++++++++++++++++++ compiler/test/TestCompilerOnCharLit.kt | 8 +++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index 5cb3cc0e8..a32d2b5fb 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -1,9 +1,6 @@ package prog8.compiler -import prog8.ast.AstToSourceCode -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.Program +import prog8.ast.* import prog8.ast.base.AstException import prog8.ast.base.Position import prog8.ast.expressions.Expression @@ -268,6 +265,9 @@ private fun processAst(programAst: Program, errors: IErrorReporter, compilerOpti println("Processing for target ${compilerOptions.compTarget.name}...") programAst.checkIdentifiers(errors, compilerOptions) errors.report() + // TODO: turning char literals into UBYTEs via an encoding should really happen in code gen - but for that we'd need DataType.CHAR + programAst.charLiteralsToUByteLiterals(errors, compilerOptions.compTarget as IStringEncoding) + errors.report() programAst.constantFold(errors, compilerOptions.compTarget) errors.report() programAst.reorderStatements(errors) diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index 47579d965..b420be300 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -1,8 +1,15 @@ package prog8.compiler.astprocessing +import prog8.ast.IStringEncoding +import prog8.ast.Node import prog8.ast.Program +import prog8.ast.base.DataType import prog8.ast.base.FatalAstException +import prog8.ast.expressions.CharLiteral +import prog8.ast.expressions.NumericLiteralValue import prog8.ast.statements.Directive +import prog8.ast.walk.AstWalker +import prog8.ast.walk.IAstModification import prog8.compiler.BeforeAsmGenerationAstChanger import prog8.compiler.CompilationOptions import prog8.compiler.IErrorReporter @@ -33,6 +40,20 @@ internal fun Program.reorderStatements(errors: IErrorReporter) { } } +internal fun Program.charLiteralsToUByteLiterals(errors: IErrorReporter, enc: IStringEncoding) { + val walker = object : AstWalker() { + override fun after(char: CharLiteral, parent: Node): Iterable { + return listOf(IAstModification.ReplaceNode( + char, + NumericLiteralValue(DataType.UBYTE, enc.encodeString(char.value.toString(), char.altEncoding)[0].toInt(), char.position), + parent + )) + } + } + walker.visit(this) + walker.applyModifications() +} + internal fun Program.addTypecasts(errors: IErrorReporter) { val caster = TypecastsAdder(this, errors) caster.visit(this) diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index 2c61ea191..9773aa000 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -40,7 +40,7 @@ class TestCompilerOnCharLit { "char literal should have been replaced by ubyte literal") val arg = funCall.args[0] as NumericLiteralValue assertEquals(DataType.UBYTE, arg.type) - assertEquals(platform.encodeString("\n", false)[0], arg.number) + assertEquals(platform.encodeString("\n", false)[0], arg.number.toShort()) // TODO: short/int/UBYTE - which should it be? } @Test @@ -67,7 +67,7 @@ class TestCompilerOnCharLit { "char literal should have been replaced by ubyte literal") val initializerValue = decl.value as NumericLiteralValue assertEquals(DataType.UBYTE, initializerValue.type) - assertEquals(platform.encodeString("\n", false)[0], initializerValue.number) + assertEquals(platform.encodeString("\n", false)[0], initializerValue.number.toShort()) // TODO: short/int/UBYTE - which should it be? } @Test @@ -87,12 +87,12 @@ class TestCompilerOnCharLit { assertEquals(DataType.UBYTE, decl.datatype) assertEquals( platform.encodeString("\n", false)[0], - (decl.value as NumericLiteralValue).number) + (decl.value as NumericLiteralValue).number.toShort()) // TODO: short/int/UBYTE - which should it be? } is NumericLiteralValue -> { assertEquals( platform.encodeString("\n", false)[0], - arg.number) + arg.number.toShort()) // TODO: short/int/UBYTE - which should it be? } else -> assertIs(funCall.args[0]) // make test fail } From ee115b33372298983131b8eb128593bb27f26b70 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 10:37:58 +0200 Subject: [PATCH 31/68] + expose #54, %asmbinary when outputDir != workingDir; also: refactor compiler tests on examples and add test helpers --- compiler/test/Helpers.kt | 156 +++++++++++++++++ compiler/test/TestCompilerOnExamples.kt | 157 ++++++++++++------ .../test/TestCompilerOnImportsAndIncludes.kt | 135 +++++++++++++++ .../test/fixtures/asmBinaryFromSameFolder.p8 | 10 ++ .../test/fixtures/asmBinaryFromSubFolder.p8 | 10 ++ .../test/fixtures/asmBinaryNonExisting.p8 | 10 ++ .../test/fixtures/asmBinaryNonReadable.p8 | 10 ++ .../test/fixtures/asmIncludeFromSameFolder.p8 | 13 ++ compiler/test/fixtures/do_nothing.asm | 7 + compiler/test/fixtures/do_nothing1.bin | 1 + compiler/test/fixtures/foo_bar.asm | 2 + compiler/test/fixtures/foo_bar.p8 | 3 + .../test/fixtures/importFromSameFolder.p8 | 9 + .../test/fixtures/subFolder/do_nothing2.bin | 1 + 14 files changed, 469 insertions(+), 55 deletions(-) create mode 100644 compiler/test/Helpers.kt create mode 100644 compiler/test/TestCompilerOnImportsAndIncludes.kt create mode 100644 compiler/test/fixtures/asmBinaryFromSameFolder.p8 create mode 100644 compiler/test/fixtures/asmBinaryFromSubFolder.p8 create mode 100644 compiler/test/fixtures/asmBinaryNonExisting.p8 create mode 100644 compiler/test/fixtures/asmBinaryNonReadable.p8 create mode 100644 compiler/test/fixtures/asmIncludeFromSameFolder.p8 create mode 100644 compiler/test/fixtures/do_nothing.asm create mode 100644 compiler/test/fixtures/do_nothing1.bin create mode 100644 compiler/test/fixtures/foo_bar.asm create mode 100644 compiler/test/fixtures/foo_bar.p8 create mode 100644 compiler/test/fixtures/importFromSameFolder.p8 create mode 100644 compiler/test/fixtures/subFolder/do_nothing2.bin diff --git a/compiler/test/Helpers.kt b/compiler/test/Helpers.kt new file mode 100644 index 000000000..0ea41bf05 --- /dev/null +++ b/compiler/test/Helpers.kt @@ -0,0 +1,156 @@ +package prog8tests.helpers + +import kotlin.test.* +import kotlin.io.path.* +import java.nio.file.Path + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.IStringEncoding +import prog8.ast.base.DataType +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue +import prog8.compiler.CompilationResult +import prog8.compiler.compileProgram +import prog8.compiler.target.ICompilationTarget + +// TODO: find a way to share with compilerAst/test/Helpers.kt, while still being able to amend it (-> compileFile(..)) + +internal fun CompilationResult.assertSuccess(description: String = ""): CompilationResult { + assertTrue(success, "expected successful compilation but failed $description") + return this +} + +internal fun CompilationResult.assertFailure(description: String = ""): CompilationResult { + assertFalse(success, "expected failure to compile but succeeded $description") + return this +} + +/** + * @see CompilationResult.assertSuccess + * @see CompilationResult.assertFailure + */ +internal fun compileFile( + platform: ICompilationTarget, + optimize: Boolean, + fileDir: Path, + fileName: String, + outputDir: Path = prog8tests.helpers.outputDir +) : CompilationResult { + val filepath = fileDir.resolve(fileName) + assumeReadableFile(filepath) + return compileProgram( + filepath, + optimize, + writeAssembly = true, + slowCodegenWarnings = false, + platform.name, + libdirs = listOf(), + outputDir + ) +} + +/** + * Takes a [sourceText] as a String, writes it to a temporary + * file and then runs the compiler on that. + * @see compileFile + */ +internal fun compileText( + platform: ICompilationTarget, + optimize: Boolean, + sourceText: String +) : CompilationResult { + val filePath = outputDir.resolve("on_the_fly_test_" + sourceText.hashCode().toUInt().toString(16) + ".p8") + // we don't assumeNotExists(filePath) - should be ok to just overwrite it + filePath.toFile().writeText(sourceText) + return compileFile(platform, optimize, filePath.parent, filePath.name) +} + + + +val workingDir : Path = Path("").absolute() // Note: Path(".") does NOT work..! +val fixturesDir : Path = workingDir.resolve("test/fixtures") +val resourcesDir : Path = workingDir.resolve("res") +val outputDir : Path = workingDir.resolve("build/tmp/test") + +fun assumeReadable(path: Path) { + assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") +} + +fun assumeReadableFile(path: Path) { + assumeReadable(path) + assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") +} + +fun assumeDirectory(path: Path) { + assertTrue(path.isDirectory(), "sanity check; should be directory: $path") +} + +fun assumeNotExists(path: Path) { + assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") +} + +fun sanityCheckDirectories(workingDirName: String? = null) { + if (workingDirName != null) + assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") + assumeDirectory(workingDir) + assumeDirectory(fixturesDir) + assumeDirectory(resourcesDir) + assumeDirectory(outputDir) +} + + +fun mapCombinations(dim1: Iterable, dim2: Iterable, combine2: (A, B) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + yield(combine2(a, b)) + }.toList() + +fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, combine3: (A, B, C) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + for (c in dim3) + yield(combine3(a, b, c)) + }.toList() + +fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, dim4: Iterable, combine4: (A, B, C, D) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + for (c in dim3) + for (d in dim4) + yield(combine4(a, b, c, d)) + }.toList() + + +val DummyEncoding = object : IStringEncoding { + override fun encodeString(str: String, altEncoding: Boolean): List { + throw Exception("just a dummy - should not be called") + } + + override fun decodeString(bytes: List, altEncoding: Boolean): String { + throw Exception("just a dummy - should not be called") + } +} + +val DummyFunctions = object : IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue( + name: String, + args: List, + position: Position, + memsizer: IMemSizer + ): NumericLiteralValue? = null + + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() +} + +val DummyMemsizer = object : IMemSizer { + override fun memorySize(dt: DataType): Int = 0 +} + diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index effd4c23d..361d5f174 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -1,80 +1,127 @@ package prog8tests -import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.DynamicTest.dynamicTest +import prog8tests.helpers.* +import kotlin.io.path.* + import prog8.compiler.compileProgram +import prog8.compiler.target.C64Target import prog8.compiler.target.Cx16Target import prog8.compiler.target.ICompilationTarget -import kotlin.io.path.Path -import kotlin.io.path.absolute -import kotlin.io.path.isDirectory -import kotlin.test.assertEquals -import kotlin.test.assertTrue /** * ATTENTION: this is just kludge! * They are not really unit tests, but rather tests of the whole process, * from source file loading all the way through to running 64tass. - * What's more: in case of failure (to compile and assemble) - which is when tests should help you - - * these tests will actually be ignored (ie. NOT fail), - * because in the code there are calls to Process.exit, making it essentially untestable. */ +//@Disabled("to save some time") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnExamples { - val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! - val examplesDir = workingDir.resolve("../examples") - val outputDir = workingDir.resolve("build/tmp/test") + private val examplesDir = workingDir.resolve("../examples") - @Test - fun testDirectoriesSanityCheck() { - assertEquals("compiler", workingDir.fileName.toString()) - assertTrue(examplesDir.isDirectory(), "sanity check; should be directory: $examplesDir") - assertTrue(outputDir.isDirectory(), "sanity check; should be directory: $outputDir") + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") + assumeDirectory(examplesDir) } // TODO: make assembly stage testable - in case of failure (eg of 64tass) it Process.exit s - fun testExample(nameWithoutExt: String, platform: ICompilationTarget, optimize: Boolean) { - val filepath = examplesDir.resolve("$nameWithoutExt.p8") - val result = compileProgram( - filepath, - optimize, - writeAssembly = true, - slowCodegenWarnings = false, - compilationTarget = platform.name, - libdirs = listOf(), - outputDir - ) - assertTrue(result.success, "${platform.name}, optimize=$optimize: \"$filepath\"") + private fun makeDynamicCompilerTest(name: String, platform: ICompilationTarget, optimize: Boolean) : DynamicTest { + val searchIn = mutableListOf(examplesDir) + if (platform == Cx16Target) { + searchIn.add(0, examplesDir.resolve("cx16")) + } + val filepath = searchIn.map { it.resolve("$name.p8") }.first { it.exists() } + val displayName = "${examplesDir.relativize(filepath)}: ${platform.name}, optimize=$optimize" + return dynamicTest(displayName) { + compileProgram( + filepath, + optimize, + writeAssembly = true, + slowCodegenWarnings = false, + compilationTarget = platform.name, + libdirs = listOf(), + outputDir + ).assertSuccess("; $displayName") + } } + @TestFactory + @Disabled + fun bothCx16AndC64() = mapCombinations( + dim1 = listOf( + "animals", + "balls", + "cube3d", + "cube3d-float", + "cube3d-gfx", + "cxlogo", + "dirlist", + "fibonacci", + "line-circle-gfx", + "line-circle-txt", + "mandelbrot", + "mandelbrot-gfx", + "numbergame", + "primes", + "rasterbars", + "screencodes", + "sorting", + "swirl", + "swirl-float", + "tehtriz", + "test", + "textelite", + ), + dim2 = listOf(Cx16Target, C64Target), + dim3 = listOf(false, true), + combine3 = ::makeDynamicCompilerTest + ) - @Test - fun test_cxLogo_noopt() { - testExample("cxlogo", Cx16Target, false) - } - @Test - fun test_cxLogo_opt() { - testExample("cxlogo", Cx16Target, true) - } - - @Test - fun test_swirl_noopt() { - testExample("swirl", Cx16Target, false) - } - @Test - fun test_swirl_opt() { - testExample("swirl", Cx16Target, true) - } - - @Test - fun test_animals_noopt() { - testExample("animals", Cx16Target, false) - } - @Test - fun test_animals_opt() { - testExample("animals", Cx16Target, true) - } + @TestFactory +// @Disabled + fun onlyC64() = mapCombinations( + dim1 = listOf( + "balloonflight", + "bdmusic", + "bdmusic-irq", + "charset", + "cube3d-sprites", + "plasma", + "sprites", + "turtle-gfx", + "wizzine", + ), + dim2 = listOf(C64Target), + dim3 = listOf(false, true), + combine3 = ::makeDynamicCompilerTest + ) + @TestFactory +// @Disabled + fun onlyCx16() = mapCombinations( + dim1 = listOf( + "vtui/testvtui", + "amiga", + "bobs", + "cobramk3-gfx", + "colorbars", + "datetime", + "highresbitmap", + "kefrenbars", + "mandelbrot-gfx-colors", + "multipalette", + "testgfx2", + ), + dim2 = listOf(Cx16Target), + dim3 = listOf(false, true), + combine3 = ::makeDynamicCompilerTest + ) } diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt new file mode 100644 index 000000000..e67600783 --- /dev/null +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -0,0 +1,135 @@ +package prog8tests + +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.DynamicTest.dynamicTest +import kotlin.test.* +import kotlin.io.path.* +import prog8tests.helpers.* + +import prog8.ast.expressions.AddressOf +import prog8.ast.expressions.IdentifierReference +import prog8.ast.expressions.StringLiteralValue +import prog8.ast.statements.FunctionCallStatement +import prog8.ast.statements.Label +import prog8.compiler.target.Cx16Target + + +/** + * ATTENTION: this is just kludge! + * They are not really unit tests, but rather tests of the whole process, + * from source file loading all the way through to running 64tass. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestCompilerOnImportsAndIncludes { + + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") + } + + @Test + fun testImportFromSameFolder() { + val filepath = fixturesDir.resolve("importFromSameFolder.p8") + val imported = fixturesDir.resolve("foo_bar.p8") + assumeReadableFile(filepath) + assumeReadableFile(imported) + + val platform = Cx16Target + val result = compileFile(platform, false, fixturesDir, filepath.name) + .assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val strLits = startSub.statements + .filterIsInstance() + .map { it.args[0] as IdentifierReference } + .map { it.targetVarDecl(program)!!.value as StringLiteralValue } + + assertEquals("main.bar", strLits[0].value) + assertEquals("foo.bar", strLits[1].value) + assertEquals("main", strLits[0].definingScope().name) + assertEquals("foo", strLits[1].definingScope().name) + } + + @Test + fun testAsmIncludeFromSameFolder() { + val filepath = fixturesDir.resolve("asmIncludeFromSameFolder.p8") + val included = fixturesDir.resolve("foo_bar.asm") + assumeReadableFile(filepath) + assumeReadableFile(included) + + val platform = Cx16Target + val result = compileFile(platform, false, fixturesDir, filepath.name) + .assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val args = startSub.statements + .filterIsInstance() + .map { it.args[0] } + + val str0 = (args[0] as IdentifierReference).targetVarDecl(program)!!.value as StringLiteralValue + assertEquals("main.bar", str0.value) + assertEquals("main", str0.definingScope().name) + + val id1 = (args[1] as AddressOf).identifier + val lbl1 = id1.targetStatement(program) as Label + assertEquals("foo_bar", lbl1.name) + assertEquals("start", lbl1.definingScope().name) + } + + @Test + fun testAsmbinaryDirectiveWithNonExistingFile() { + val p8Path = fixturesDir.resolve("asmbinaryNonExisting.p8") + val binPath = fixturesDir.resolve("i_do_not_exist.bin") + assumeReadableFile(p8Path) + assumeNotExists(binPath) + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertFailure() + } + + @Test + fun testAsmbinaryDirectiveWithNonReadableFile() { + val p8Path = fixturesDir.resolve("asmbinaryNonReadable.p8") + val binPath = fixturesDir.resolve("subFolder") + assumeReadableFile(p8Path) + assumeDirectory(binPath) + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertFailure() + } + + @TestFactory + fun asmbinaryDirectiveWithExistingBinFile(): Iterable = + listOf( + Triple("same ", "asmBinaryFromSameFolder.p8", "do_nothing1.bin"), + Triple("sub", "asmBinaryFromSubFolder.p8", "subFolder/do_nothing2.bin"), + ).map { + val (where, p8Str, binStr) = it + val p8Path = fixturesDir.resolve(p8Str) + val binPath = fixturesDir.resolve(binStr) + val displayName = "%asmbinary from ${where}folder" + dynamicTest(displayName) { + assumeReadableFile(p8Path) + assumeReadableFile(binPath) + assertNotEquals( // the bug we're testing for (#??) was hidden if outputDir == workinDir + workingDir.normalize().toAbsolutePath(), + outputDir.normalize().toAbsolutePath(), + "sanity check: workingDir and outputDir should not be the same folder") + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertSuccess( + "argument to assembler directive .binary " + + "should be relative to the generated .asm file (in output dir), " + + "NOT relative to .p8 neither current working dir" + ) + } + } + + } + diff --git a/compiler/test/fixtures/asmBinaryFromSameFolder.p8 b/compiler/test/fixtures/asmBinaryFromSameFolder.p8 new file mode 100644 index 000000000..d48a8c6b0 --- /dev/null +++ b/compiler/test/fixtures/asmBinaryFromSameFolder.p8 @@ -0,0 +1,10 @@ +main { + sub start() { + stuff.do_nothing() + } +} + +stuff $1000 { + romsub $1000 = do_nothing() + %asmbinary "do_nothing1.bin", 0 +} \ No newline at end of file diff --git a/compiler/test/fixtures/asmBinaryFromSubFolder.p8 b/compiler/test/fixtures/asmBinaryFromSubFolder.p8 new file mode 100644 index 000000000..5b48b7892 --- /dev/null +++ b/compiler/test/fixtures/asmBinaryFromSubFolder.p8 @@ -0,0 +1,10 @@ +main { + sub start() { + stuff.do_nothing() + } +} + +stuff $1000 { + romsub $1000 = do_nothing() + %asmbinary "subFolder/do_nothing2.bin", 0 +} \ No newline at end of file diff --git a/compiler/test/fixtures/asmBinaryNonExisting.p8 b/compiler/test/fixtures/asmBinaryNonExisting.p8 new file mode 100644 index 000000000..2ff4e2f46 --- /dev/null +++ b/compiler/test/fixtures/asmBinaryNonExisting.p8 @@ -0,0 +1,10 @@ +main { + sub start() { + stuff.do_nothing() + } +} + +stuff $1000 { + romsub $1000 = do_nothing() + %asmbinary "i_do_not_exist.bin", 0 +} \ No newline at end of file diff --git a/compiler/test/fixtures/asmBinaryNonReadable.p8 b/compiler/test/fixtures/asmBinaryNonReadable.p8 new file mode 100644 index 000000000..3b7f63bfc --- /dev/null +++ b/compiler/test/fixtures/asmBinaryNonReadable.p8 @@ -0,0 +1,10 @@ +main { + sub start() { + stuff.do_nothing() + } +} + +stuff $1000 { + romsub $1000 = do_nothing() + %asmbinary "subFolder", 0 +} \ No newline at end of file diff --git a/compiler/test/fixtures/asmIncludeFromSameFolder.p8 b/compiler/test/fixtures/asmIncludeFromSameFolder.p8 new file mode 100644 index 000000000..df25f6e78 --- /dev/null +++ b/compiler/test/fixtures/asmIncludeFromSameFolder.p8 @@ -0,0 +1,13 @@ +%import textio +main { + str myBar = "main.bar" +;foo_bar: +; %asminclude "foo_bar.asm" ; FIXME: should be accessible from inside start() but give assembler error + sub start() { + txt.print(myBar) + txt.print(&foo_bar) + return +foo_bar: + %asminclude "foo_bar.asm" + } +} diff --git a/compiler/test/fixtures/do_nothing.asm b/compiler/test/fixtures/do_nothing.asm new file mode 100644 index 000000000..6bfed22b6 --- /dev/null +++ b/compiler/test/fixtures/do_nothing.asm @@ -0,0 +1,7 @@ +; this is the source code for do_nothing1.bin and subFolder/do_nothing2.bin +; command lines: +; 64tass --ascii --nostart do_nothing.asm --output do_nothing1.bin +; 64tass --ascii --nostart do_nothing.asm --output subFolder/do_nothing2.bin +*=0 + rts + diff --git a/compiler/test/fixtures/do_nothing1.bin b/compiler/test/fixtures/do_nothing1.bin new file mode 100644 index 000000000..64845fb76 --- /dev/null +++ b/compiler/test/fixtures/do_nothing1.bin @@ -0,0 +1 @@ +` \ No newline at end of file diff --git a/compiler/test/fixtures/foo_bar.asm b/compiler/test/fixtures/foo_bar.asm new file mode 100644 index 000000000..d415b6923 --- /dev/null +++ b/compiler/test/fixtures/foo_bar.asm @@ -0,0 +1,2 @@ +bar .text "foo.bar",0 + diff --git a/compiler/test/fixtures/foo_bar.p8 b/compiler/test/fixtures/foo_bar.p8 new file mode 100644 index 000000000..e60fa8199 --- /dev/null +++ b/compiler/test/fixtures/foo_bar.p8 @@ -0,0 +1,3 @@ +foo { + str bar = "foo.bar" +} diff --git a/compiler/test/fixtures/importFromSameFolder.p8 b/compiler/test/fixtures/importFromSameFolder.p8 new file mode 100644 index 000000000..3f233b10d --- /dev/null +++ b/compiler/test/fixtures/importFromSameFolder.p8 @@ -0,0 +1,9 @@ +%import textio +%import foo_bar +main { + str myBar = "main.bar" + sub start() { + txt.print(myBar) + txt.print(foo.bar) + } +} diff --git a/compiler/test/fixtures/subFolder/do_nothing2.bin b/compiler/test/fixtures/subFolder/do_nothing2.bin new file mode 100644 index 000000000..64845fb76 --- /dev/null +++ b/compiler/test/fixtures/subFolder/do_nothing2.bin @@ -0,0 +1 @@ +` \ No newline at end of file From 23c99002c01df8bf174d200bc1663da8198eaa44 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 12:04:49 +0200 Subject: [PATCH 32/68] * fix #54 / step 1: relativize threw IllegalArgumentException if called on non-absolute path with absolute path as argument ("different type of path") --- .../prog8/compiler/target/cpu6502/codegen/AsmGen.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt index 212911571..6513f0deb 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt @@ -18,6 +18,7 @@ import java.nio.file.Paths import java.time.LocalDate import java.time.LocalDateTime import java.util.* +import kotlin.io.path.absolute import kotlin.math.absoluteValue @@ -1315,11 +1316,14 @@ $repeatLabel lda $counterVar assemblyLines.add(sourcecode.trimEnd().trimStart('\n')) } "%asmbinary" -> { + val includedName = stmt.args[0].str!! val offset = if(stmt.args.size>1) ", ${stmt.args[1].int}" else "" val length = if(stmt.args.size>2) ", ${stmt.args[2].int}" else "" - val includedSourcePath = stmt.definingModule().source.resolveSibling(stmt.args[0].str) - val relPath = Paths.get("").relativize(includedSourcePath) - out(" .binary \"$relPath\" $offset $length") + val includedPath = stmt.definingModule().source.resolveSibling(includedName) + val pathForAssembler = Paths.get("") + .absolute() // avoid IllegalArgumentExc in case non-absolute path .relativize(absolute path) + .relativize(includedPath) + out(" .binary \"$pathForAssembler\" $offset $length") } "%breakpoint" -> { val label = "_prog8_breakpoint_${breakpointLabels.size+1}" From 402884b5ce6b6d13222c5d1eb50cbce79325c1ff Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 12:17:31 +0200 Subject: [PATCH 33/68] * fix #54 / step 2: the path stated with assembler directive .binary must be *relative to the .asm file*, not the working directory --- .../compiler/target/cpu6502/codegen/AsmGen.kt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt index 6513f0deb..2a4b30b58 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt @@ -1309,19 +1309,26 @@ $repeatLabel lda $counterVar } } + /** + * TODO: %asminclude and %asmbinary should be done earlier than code gen (-> put content into AST) + */ private fun translate(stmt: Directive) { when(stmt.directive) { "%asminclude" -> { - val sourcecode = loadAsmIncludeFile(stmt.args[0].str!!, stmt.definingModule().source) + val includedName = stmt.args[0].str!! + val sourcePath = stmt.definingModule().source // FIXME: %asminclude inside non-library, non-filesystem module + val sourcecode = loadAsmIncludeFile(includedName, sourcePath) assemblyLines.add(sourcecode.trimEnd().trimStart('\n')) } "%asmbinary" -> { val includedName = stmt.args[0].str!! val offset = if(stmt.args.size>1) ", ${stmt.args[1].int}" else "" val length = if(stmt.args.size>2) ", ${stmt.args[2].int}" else "" - val includedPath = stmt.definingModule().source.resolveSibling(includedName) - val pathForAssembler = Paths.get("") - .absolute() // avoid IllegalArgumentExc in case non-absolute path .relativize(absolute path) + val sourcePath = stmt.definingModule().source // FIXME: %asmbinary inside non-library, non-filesystem module + val includedPath = sourcePath.resolveSibling(includedName) + + val pathForAssembler = outputDir // 64tass needs the path *relative to the .asm file* + .absolute() // avoid IllegalArgumentExc due to non-absolute path .relativize(absolute path) .relativize(includedPath) out(" .binary \"$pathForAssembler\" $offset $length") } From b2c6274f7453ccdef7895bc41459b5af216beace Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 12:40:50 +0200 Subject: [PATCH 34/68] * fix #54 / step 3: avoid some (= not all) complaints re the .binary filename 64tass still had/has. Actually, I don't quite understand why it still says "not the real name of the file". The 64tass docs say: > -Wno-portable > Don't warn about source portability problems. > These cross platform development annoyances are checked for: > * Case insensitive use of file names or use of short names. > * Use of backslashes for path separation instead of forward slashes. > * Use of reserved characters in file names. > * Absolute paths --- compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt | 4 +++- compiler/test/TestCompilerOnImportsAndIncludes.kt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt index 2a4b30b58..27ca32e1a 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/AsmGen.kt @@ -1327,9 +1327,11 @@ $repeatLabel lda $counterVar val sourcePath = stmt.definingModule().source // FIXME: %asmbinary inside non-library, non-filesystem module val includedPath = sourcePath.resolveSibling(includedName) - val pathForAssembler = outputDir // 64tass needs the path *relative to the .asm file* + val pathForAssembler = outputDir // #54: 64tass needs the path *relative to the .asm file* .absolute() // avoid IllegalArgumentExc due to non-absolute path .relativize(absolute path) .relativize(includedPath) + .normalize() // avoid assembler warnings (-Wportable; only some, not all) + .toString().replace('\\', '/') out(" .binary \"$pathForAssembler\" $offset $length") } "%breakpoint" -> { diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index e67600783..d166f92d3 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -117,7 +117,7 @@ class TestCompilerOnImportsAndIncludes { dynamicTest(displayName) { assumeReadableFile(p8Path) assumeReadableFile(binPath) - assertNotEquals( // the bug we're testing for (#??) was hidden if outputDir == workinDir + assertNotEquals( // the bug we're testing for (#54) was hidden if outputDir == workinDir workingDir.normalize().toAbsolutePath(), outputDir.normalize().toAbsolutePath(), "sanity check: workingDir and outputDir should not be the same folder") From 34ba07ee3b874e70e59e5a72dcb627b95d0b227f Mon Sep 17 00:00:00 2001 From: meisl Date: Tue, 13 Jul 2021 09:47:31 +0200 Subject: [PATCH 35/68] + expose #55: float[] initializer as range where no array size is stated --- compiler/test/TestCompilerOnCharLit.kt | 37 ++- compiler/test/TestCompilerOnRanges.kt | 267 ++++++++++++++++++ .../test/fixtures/charConstAsRomsubArg.p8 | 7 - compiler/test/fixtures/charLitAsRomsubArg.p8 | 6 - compiler/test/fixtures/charVarAsRomsubArg.p8 | 7 - compilerAst/test/TestProg8Parser.kt | 51 +++- 6 files changed, 347 insertions(+), 28 deletions(-) create mode 100644 compiler/test/TestCompilerOnRanges.kt delete mode 100644 compiler/test/fixtures/charConstAsRomsubArg.p8 delete mode 100644 compiler/test/fixtures/charLitAsRomsubArg.p8 delete mode 100644 compiler/test/fixtures/charVarAsRomsubArg.p8 diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index d53beb743..08d50c272 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -11,6 +11,8 @@ import prog8.ast.base.DataType import prog8.ast.base.VarDeclType import prog8.ast.expressions.IdentifierReference import prog8.ast.expressions.NumericLiteralValue +import prog8.ast.expressions.RangeExpr +import prog8.ast.statements.ForLoop import prog8.compiler.target.Cx16Target @@ -30,8 +32,14 @@ class TestCompilerOnCharLit { @Test fun testCharLitAsRomsubArg() { val platform = Cx16Target - val result = compileFile(platform, optimize = false, fixturesDir, "charLitAsRomsubArg.p8") - .assertSuccess() + val result = compileText(platform, false, """ + main { + romsub ${"$"}FFD2 = chrout(ubyte ch @ A) + sub start() { + chrout('\n') + } + } + """).assertSuccess() val program = result.programAst val startSub = program.entrypoint() @@ -47,8 +55,15 @@ class TestCompilerOnCharLit { @Test fun testCharVarAsRomsubArg() { val platform = Cx16Target - val result = compileFile(platform, optimize = false, fixturesDir, "charVarAsRomsubArg.p8") - .assertSuccess() + val result = compileText(platform, false, """ + main { + romsub ${"$"}FFD2 = chrout(ubyte ch @ A) + sub start() { + ubyte ch = '\n' + chrout(ch) + } + } + """).assertSuccess() val program = result.programAst val startSub = program.entrypoint() @@ -75,8 +90,15 @@ class TestCompilerOnCharLit { @Test fun testCharConstAsRomsubArg() { val platform = Cx16Target - val result = compileFile(platform, optimize = false, fixturesDir,"charConstAsRomsubArg.p8") - .assertSuccess() + val result = compileText(platform, false, """ + main { + romsub ${"$"}FFD2 = chrout(ubyte ch @ A) + sub start() { + const ubyte ch = '\n' + chrout(ch) + } + } + """).assertSuccess() val program = result.programAst val startSub = program.entrypoint() @@ -99,6 +121,7 @@ class TestCompilerOnCharLit { } else -> assertIs(funCall.args[0]) // make test fail } - } + } + diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt new file mode 100644 index 000000000..76e8d1b32 --- /dev/null +++ b/compiler/test/TestCompilerOnRanges.kt @@ -0,0 +1,267 @@ +package prog8tests + +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.DynamicTest.dynamicTest +import kotlin.test.* +import prog8tests.helpers.* + +import prog8.ast.base.DataType +import prog8.ast.expressions.* +import prog8.ast.statements.ForLoop +import prog8.ast.statements.Subroutine +import prog8.ast.statements.VarDecl +import prog8.compiler.target.C64Target +import prog8.compiler.target.Cx16Target +import prog8.optimizer.ConstantIdentifierReplacer + + +/** + * ATTENTION: this is just kludge! + * They are not really unit tests, but rather tests of the whole process, + * from source file loading all the way through to running 64tass. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestCompilerOnRanges { + + @BeforeAll + fun setUp() { + sanityCheckDirectories("compiler") + } + + @Test + fun testUByteArrayInitializerWithRange_char_to_char() { + val platform = Cx16Target + val result = compileText(platform, true, """ + main { + sub start() { + ubyte[] cs = @'a' to 'z' ; values are computed at compile time + cs[0] = 23 ; keep optimizer from removing it + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val decl = startSub + .statements.filterIsInstance()[0] + val rhsValues = (decl.value as ArrayLiteralValue) + .value // Array + .map { (it as NumericLiteralValue).number.toInt() } + val expectedStart = platform.encodeString("a", true)[0].toInt() + val expectedEnd = platform.encodeString("z", false)[0].toInt() + val expectedStr = "$expectedStart .. $expectedEnd" + + val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}" + assertEquals(expectedStr, actualStr,".first .. .last") + assertEquals(expectedEnd - expectedStart + 1, rhsValues.last() - rhsValues.first() + 1, "rangeExpr.size()") + } + + @Test +// @Disabled("bug in ConstantIdentifierReplacer.before(VarDecl)@decl.datatype==ARRAY_F") + fun testFloatArrayInitializerWithRange_char_to_char() { + val platform = C64Target + val result = compileText(platform, optimize = false, """ + %option enable_floats + main { + sub start() { + float[] cs = @'a' to 'z' ; values are computed at compile time + cs[0] = 23 ; keep optimizer from removing it + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val decl = startSub + .statements.filterIsInstance()[0] + val rhsValues = (decl.value as ArrayLiteralValue) + .value // Array + .map { (it as NumericLiteralValue).number.toInt() } + val expectedStart = platform.encodeString("a", true)[0].toInt() + val expectedEnd = platform.encodeString("z", false)[0].toInt() + val expectedStr = "$expectedStart .. $expectedEnd" + + val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}" + assertEquals(expectedStr, actualStr,".first .. .last") + assertEquals(expectedEnd - expectedStart + 1, rhsValues.size, "rangeExpr.size()") + } + + inline fun Subroutine.decl(varName: String): VarDecl { + return statements.filterIsInstance() + .first { it.name == varName } + } + inline fun VarDecl.rhs() : T { + return value as T + } + inline fun ArrayLiteralValue.elements() : List { + return value.map { it as T } + } + + fun assertEndpoints(expFirst: N, expLast: N, actual: Iterable, msg: String = ".first .. .last") { + val expectedStr = "$expFirst .. $expLast" + val actualStr = "${actual.first()} .. ${actual.last()}" + assertEquals(expectedStr, actualStr,".first .. .last") + } + + + @TestFactory + fun floatArrayInitializerWithRange() = mapCombinations( + dim1 = listOf("", "42", "41"), // sizeInDecl + dim2 = listOf("%option enable_floats", ""), // optEnableFloats + dim3 = listOf(Cx16Target, C64Target), // platform + dim4 = listOf(false, true), // optimize + combine4 = { sizeInDecl, optEnableFloats, platform, optimize -> + val displayName = + when (sizeInDecl) { + "" -> "no" + "42" -> "correct" + else -> "wrong" + } + " array size given" + + ", " + (if (optEnableFloats == "") "without" else "with") + " %option enable_floats" + + ", ${platform.name}, optimize: $optimize" + dynamicTest(displayName) { + val result = compileText(platform, optimize, """ + $optEnableFloats + main { + sub start() { + float[$sizeInDecl] cs = 1 to 42 ; values are computed at compile time + cs[0] = 23 ; keep optimizer from removing it + } + } + """) + if ((sizeInDecl == "42") && (optEnableFloats != "")){ + result.assertSuccess() + } else { + result.assertFailure() + } + } + } + ) + + @Test + fun testForLoopWithRange_char_to_char() { + val platform = Cx16Target + val result = compileText(platform, true, """ + main { + sub start() { + ubyte i + for i in @'a' to 'f' { + i += i ; keep optimizer from removing it + } + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val iterable = startSub + .statements.filterIsInstance() + .map { it.iterable }[0] + val rangeExpr = iterable as RangeExpr + + val expectedStart = platform.encodeString("a", true)[0].toInt() + val expectedEnd = platform.encodeString("f", false)[0].toInt() + val expectedStr = "$expectedStart .. $expectedEnd" + + val intProgression = rangeExpr.toConstantIntegerRange() + val actualStr = "${intProgression?.first} .. ${intProgression?.last}" + assertEquals(expectedStr, actualStr,".first .. .last") + assertEquals(expectedEnd - expectedStart + 1, rangeExpr.size(), "rangeExpr.size()") + } + + @Test + fun testForLoopWithRange_bool_to_bool() { + val result = compileText(Cx16Target, true, """ + main { + sub start() { + ubyte i + for i in false to true { + i += i ; keep optimizer from removing it + } + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val rangeExpr = startSub + .statements.filterIsInstance() + .map { it.iterable } + .filterIsInstance()[0] + + assertEquals(2, rangeExpr.size()) + val intProgression = rangeExpr.toConstantIntegerRange() + assertEquals(0, intProgression?.first) + assertEquals(1, intProgression?.last) + } + + @Test + fun testForLoopWithRange_ubyte_to_ubyte() { + val result = compileText(Cx16Target, true, """ + main { + sub start() { + ubyte i + for i in 1 to 9 { + i += i ; keep optimizer from removing it + } + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val rangeExpr = startSub + .statements.filterIsInstance() + .map { it.iterable } + .filterIsInstance()[0] + + assertEquals(9, rangeExpr.size()) + val intProgression = rangeExpr.toConstantIntegerRange() + assertEquals(1, intProgression?.first) + assertEquals(9, intProgression?.last) + } + + @Test + fun testForLoopWithRange_str_downto_str() { + val result = compileText(Cx16Target, true, """ + main { + sub start() { + ubyte i + for i in "start" downto "end" { + i += i ; keep optimizer from removing it + } + } + } + """).assertFailure() + //TODO("test exact compile error(s)") + } + + @Test + fun testForLoopWithIterable_str() { + val result = compileText(Cx16Target, false, """ + main { + sub start() { + ubyte i + for i in "something" { + i += i ; keep optimizer from removing it + } + } + } + """).assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val iterable = startSub + .statements.filterIsInstance() + .map { it.iterable } + .filterIsInstance()[0] + + assertEquals(DataType.STR, iterable.inferType(program).typeOrElse(DataType.UNDEFINED)) + } + +} + diff --git a/compiler/test/fixtures/charConstAsRomsubArg.p8 b/compiler/test/fixtures/charConstAsRomsubArg.p8 deleted file mode 100644 index 838149365..000000000 --- a/compiler/test/fixtures/charConstAsRomsubArg.p8 +++ /dev/null @@ -1,7 +0,0 @@ -main { - romsub $FFD2 = chrout(ubyte ch @ A) - sub start() { - const ubyte ch = '\n' - chrout(ch) - } -} diff --git a/compiler/test/fixtures/charLitAsRomsubArg.p8 b/compiler/test/fixtures/charLitAsRomsubArg.p8 deleted file mode 100644 index ebcfa426e..000000000 --- a/compiler/test/fixtures/charLitAsRomsubArg.p8 +++ /dev/null @@ -1,6 +0,0 @@ -main { - romsub $FFD2 = chrout(ubyte ch @ A) - sub start() { - chrout('\n') - } -} diff --git a/compiler/test/fixtures/charVarAsRomsubArg.p8 b/compiler/test/fixtures/charVarAsRomsubArg.p8 deleted file mode 100644 index e04462345..000000000 --- a/compiler/test/fixtures/charVarAsRomsubArg.p8 +++ /dev/null @@ -1,7 +0,0 @@ -main { - romsub $FFD2 = chrout(ubyte ch @ A) - sub start() { - ubyte ch = '\n' - chrout(ch) - } -} diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 9826bf908..622198561 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -14,7 +14,7 @@ import prog8.parser.SourceCode import prog8.ast.* import prog8.ast.statements.* import prog8.ast.base.Position -import prog8.ast.expressions.CharLiteral +import prog8.ast.expressions.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -431,4 +431,53 @@ class TestProg8Parser { assertEquals('x', rhs.value, "char literal's .value") assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") } + + + @Test + fun testForloop() { + val module = parseModule(SourceCode.of(""" + main { + sub start() { + ubyte ub + for ub in "start" downto "end" { ; #0 + } + for ub in "something" { ; #1 + } + for ub in @'a' to 'f' { ; #2 + } + for ub in false to true { ; #3 + } + for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check! + } + } + } + """)) + val iterables = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance() + .map { it.iterable } + + assertEquals(5, iterables.size) + + val it0 = iterables[0] as RangeExpr + assertIs(it0.from, "parser should leave it as is") + assertIs(it0.to, "parser should leave it as is") + + val it1 = iterables[1] as StringLiteralValue + assertEquals("something", it1.value, "parser should leave it as is") + + val it2 = iterables[2] as RangeExpr + assertIs(it2.from, "parser should leave it as is") + assertIs(it2.to, "parser should leave it as is") + + val it3 = iterables[3] as RangeExpr + // TODO: intro BoolLiteral + assertIs(it3.from, "parser should leave it as is") + assertIs(it3.to, "parser should leave it as is") + + val it4 = iterables[4] as RangeExpr + assertIs(it4.from, "parser should leave it as is") + assertIs(it4.to, "parser should leave it as is") + } } From 3f6f25e06f144398826ca2d6b155f39ba7215e71 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 17:12:16 +0200 Subject: [PATCH 36/68] * @Disable tests re unsolved #55, "float[] initializer with range and no explicit array size" --- compiler/test/TestCompilerOnRanges.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index 76e8d1b32..ee11c39ff 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -61,7 +61,7 @@ class TestCompilerOnRanges { } @Test -// @Disabled("bug in ConstantIdentifierReplacer.before(VarDecl)@decl.datatype==ARRAY_F") + @Disabled("#55: bug in ConstantIdentifierReplacer.before(VarDecl)@decl.datatype==ARRAY_F") fun testFloatArrayInitializerWithRange_char_to_char() { val platform = C64Target val result = compileText(platform, optimize = false, """ @@ -109,6 +109,7 @@ class TestCompilerOnRanges { @TestFactory + @Disabled("#55") fun floatArrayInitializerWithRange() = mapCombinations( dim1 = listOf("", "42", "41"), // sizeInDecl dim2 = listOf("%option enable_floats", ""), // optEnableFloats From 48d3abc1feef35d81e4155a137f3390e30741ebb Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 20:45:17 +0200 Subject: [PATCH 37/68] * refactor RangeExpr, step 1: remove IStringEncoding as ctor arg and instead put it as arg to the two methods that actually depend on it: toConstantIntegerRange and size (as *it* calls the former) --- .../target/cpu6502/codegen/ForLoopsAsmGen.kt | 2 +- .../optimizer/ConstantFoldingOptimizer.kt | 2 +- .../optimizer/ConstantIdentifierReplacer.kt | 8 +++---- .../src/prog8/optimizer/StatementOptimizer.kt | 2 +- compiler/test/TestCompilerOnRanges.kt | 21 ++++++++++--------- compilerAst/src/prog8/ast/AstToplevel.kt | 3 --- .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 8 ++----- .../prog8/ast/expressions/AstExpressions.kt | 7 +++---- 8 files changed, 23 insertions(+), 30 deletions(-) diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt index 24e5229ec..1aa1a8bff 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt @@ -19,7 +19,7 @@ internal class ForLoopsAsmGen(private val program: Program, private val asmgen: throw AssemblyError("unknown dt") when(stmt.iterable) { is RangeExpr -> { - val range = (stmt.iterable as RangeExpr).toConstantIntegerRange() + val range = (stmt.iterable as RangeExpr).toConstantIntegerRange(asmgen.options.compTarget) if(range==null) { translateForOverNonconstRange(stmt, iterableDt.typeOrElse(DataType.UNDEFINED), stmt.iterable as RangeExpr) } else { diff --git a/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt b/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt index d0315a620..abe9ce509 100644 --- a/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt +++ b/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt @@ -223,7 +223,7 @@ internal class ConstantFoldingOptimizer(private val program: Program, private va range.step } - return RangeExpr(fromCast.valueOrZero(), toCast.valueOrZero(), newStep, compTarget, range.position) + return RangeExpr(fromCast.valueOrZero(), toCast.valueOrZero(), newStep, range.position) } // adjust the datatype of a range expression in for loops to the loop variable. diff --git a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt index aae516238..f45bfd4a1 100644 --- a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt +++ b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt @@ -129,9 +129,9 @@ internal class ConstantIdentifierReplacer(private val program: Program, private if(rangeExpr!=null) { // convert the initializer range expression to an actual array val declArraySize = decl.arraysize?.constIndex() - if(declArraySize!=null && declArraySize!=rangeExpr.size()) + if(declArraySize!=null && declArraySize!=rangeExpr.size(compTarget)) errors.err("range expression size doesn't match declared array size", decl.value?.position!!) - val constRange = rangeExpr.toConstantIntegerRange() + val constRange = rangeExpr.toConstantIntegerRange(compTarget) if(constRange!=null) { val eltType = rangeExpr.inferType(program).typeOrElse(DataType.UBYTE) val newValue = if(eltType in ByteDatatypes) { @@ -184,9 +184,9 @@ internal class ConstantIdentifierReplacer(private val program: Program, private if(rangeExpr!=null) { // convert the initializer range expression to an actual array of floats val declArraySize = decl.arraysize?.constIndex() - if(declArraySize!=null && declArraySize!=rangeExpr.size()) + if(declArraySize!=null && declArraySize!=rangeExpr.size(compTarget)) errors.err("range expression size doesn't match declared array size", decl.value?.position!!) - val constRange = rangeExpr.toConstantIntegerRange() + val constRange = rangeExpr.toConstantIntegerRange(compTarget) if(constRange!=null) { val newValue = ArrayLiteralValue(InferredTypes.InferredType.known(DataType.ARRAY_F), constRange.map { NumericLiteralValue(DataType.FLOAT, it.toDouble(), decl.value!!.position) }.toTypedArray(), diff --git a/compiler/src/prog8/optimizer/StatementOptimizer.kt b/compiler/src/prog8/optimizer/StatementOptimizer.kt index 413b6e2fd..af7455e22 100644 --- a/compiler/src/prog8/optimizer/StatementOptimizer.kt +++ b/compiler/src/prog8/optimizer/StatementOptimizer.kt @@ -162,7 +162,7 @@ internal class StatementOptimizer(private val program: Program, val range = forLoop.iterable as? RangeExpr if(range!=null) { - if(range.size()==1) { + if (range.size(compTarget) == 1) { // for loop over a (constant) range of just a single value-- optimize the loop away // loopvar/reg = range value , follow by block val scope = AnonymousScope(mutableListOf(), forLoop.position) diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index ee11c39ff..18e7bd16f 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -16,7 +16,6 @@ import prog8.ast.statements.Subroutine import prog8.ast.statements.VarDecl import prog8.compiler.target.C64Target import prog8.compiler.target.Cx16Target -import prog8.optimizer.ConstantIdentifierReplacer /** @@ -146,7 +145,7 @@ class TestCompilerOnRanges { @Test fun testForLoopWithRange_char_to_char() { val platform = Cx16Target - val result = compileText(platform, true, """ + val result = compileText(platform, optimize = true, """ main { sub start() { ubyte i @@ -168,15 +167,16 @@ class TestCompilerOnRanges { val expectedEnd = platform.encodeString("f", false)[0].toInt() val expectedStr = "$expectedStart .. $expectedEnd" - val intProgression = rangeExpr.toConstantIntegerRange() + val intProgression = rangeExpr.toConstantIntegerRange(platform) val actualStr = "${intProgression?.first} .. ${intProgression?.last}" assertEquals(expectedStr, actualStr,".first .. .last") - assertEquals(expectedEnd - expectedStart + 1, rangeExpr.size(), "rangeExpr.size()") + assertEquals(expectedEnd - expectedStart + 1, rangeExpr.size(platform), "rangeExpr.size()") } @Test fun testForLoopWithRange_bool_to_bool() { - val result = compileText(Cx16Target, true, """ + val platform = Cx16Target + val result = compileText(platform, optimize = true, """ main { sub start() { ubyte i @@ -194,15 +194,16 @@ class TestCompilerOnRanges { .map { it.iterable } .filterIsInstance()[0] - assertEquals(2, rangeExpr.size()) - val intProgression = rangeExpr.toConstantIntegerRange() + assertEquals(2, rangeExpr.size(platform)) + val intProgression = rangeExpr.toConstantIntegerRange(platform) assertEquals(0, intProgression?.first) assertEquals(1, intProgression?.last) } @Test fun testForLoopWithRange_ubyte_to_ubyte() { - val result = compileText(Cx16Target, true, """ + val platform = Cx16Target + val result = compileText(platform, optimize = true, """ main { sub start() { ubyte i @@ -220,8 +221,8 @@ class TestCompilerOnRanges { .map { it.iterable } .filterIsInstance()[0] - assertEquals(9, rangeExpr.size()) - val intProgression = rangeExpr.toConstantIntegerRange() + assertEquals(9, rangeExpr.size(platform)) + val intProgression = rangeExpr.toConstantIntegerRange(platform) assertEquals(1, intProgression?.first) assertEquals(9, intProgression?.last) } diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 81436cdac..dfbf16310 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -6,9 +6,6 @@ import prog8.ast.statements.* import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstVisitor import prog8.parser.SourceCode -import java.nio.file.Path -import java.nio.file.Paths -import kotlin.io.path.name import kotlin.math.abs const val internedStringsModuleName = "prog8_interned_strings" diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index c2c81dd1c..ed6a10c2c 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -1,6 +1,5 @@ package prog8.ast.antlr -import org.antlr.v4.runtime.IntStream import org.antlr.v4.runtime.ParserRuleContext import org.antlr.v4.runtime.tree.TerminalNode import prog8.ast.IStringEncoding @@ -10,9 +9,6 @@ import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.parser.Prog8ANTLRParser import prog8.parser.SourceCode -import java.io.CharConversionException -import java.io.File -import java.nio.file.Path /***************** Antlr Extension methods to create AST ****************/ @@ -462,7 +458,7 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) if (rangefrom!=null && rangeto!=null) { val defaultstep = if(rto.text == "to") 1 else -1 val step = rangestep?.toAst(encoding) ?: NumericLiteralValue(DataType.UBYTE, defaultstep, toPosition()) - return RangeExpr(rangefrom.toAst(encoding), rangeto.toAst(encoding), step, encoding, toPosition()) + return RangeExpr(rangefrom.toAst(encoding), rangeto.toAst(encoding), step, toPosition()) } if(childCount==3 && children[0].text=="(" && children[2].text==")") @@ -581,7 +577,7 @@ private fun Prog8ANTLRParser.UntilloopContext.toAst(encoding: IStringEncoding): private fun Prog8ANTLRParser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { val condition = expression().toAst(encoding) - val choices = this.when_choice()?.map { it.toAst(encoding) }?.toMutableList() ?: mutableListOf() + val choices = this.when_choice()?.map { it.toAst(encoding) }?.toMutableList() ?: mutableListOf() return WhenStatement(condition, choices, toPosition()) } diff --git a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt index d9aa47362..48205ff30 100644 --- a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt +++ b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt @@ -667,7 +667,6 @@ class ArrayLiteralValue(val type: InferredTypes.InferredType, // inferred be class RangeExpr(var from: Expression, var to: Expression, var step: Expression, - private val encoding: IStringEncoding, override val position: Position) : Expression() { override lateinit var parent: Node @@ -720,15 +719,15 @@ class RangeExpr(var from: Expression, return "RangeExpr(from $from, to $to, step $step, pos=$position)" } - fun size(): Int? { + fun size(encoding: IStringEncoding): Int? { val fromLv = (from as? NumericLiteralValue) val toLv = (to as? NumericLiteralValue) if(fromLv==null || toLv==null) return null - return toConstantIntegerRange()?.count() + return toConstantIntegerRange(encoding)?.count() } - fun toConstantIntegerRange(): IntProgression? { + fun toConstantIntegerRange(encoding: IStringEncoding): IntProgression? { val fromVal: Int val toVal: Int val fromString = from as? StringLiteralValue From 522bf91c301ac41bb336837a09bfce025490e547 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 21:13:34 +0200 Subject: [PATCH 38/68] * refactor RangeExpr, step 2: make toConstantIntegerRange and size *extension methods* and move them to compiler/astprocessing/AstExtensions.kt (along with the simple helper makeRange). They are in fact *only* used from the compiler module - strong indication that they actually belong there. --- .../compiler/astprocessing/AstExtensions.kt | 50 ++++++++++++++++++- .../target/cpu6502/codegen/ForLoopsAsmGen.kt | 1 + .../optimizer/ConstantIdentifierReplacer.kt | 2 + .../src/prog8/optimizer/StatementOptimizer.kt | 1 + compiler/test/TestCompilerOnRanges.kt | 2 + .../prog8/ast/expressions/AstExpressions.kt | 44 ---------------- .../src/prog8/parser/PetsciiEncoding.kt | 2 +- .../src/prog8/parser/ThrowTodoEncoding.kt | 2 +- 8 files changed, 56 insertions(+), 48 deletions(-) diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index b420be300..3eb3760a6 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -5,8 +5,7 @@ import prog8.ast.Node import prog8.ast.Program import prog8.ast.base.DataType import prog8.ast.base.FatalAstException -import prog8.ast.expressions.CharLiteral -import prog8.ast.expressions.NumericLiteralValue +import prog8.ast.expressions.* import prog8.ast.statements.Directive import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstModification @@ -14,6 +13,53 @@ import prog8.compiler.BeforeAsmGenerationAstChanger import prog8.compiler.CompilationOptions import prog8.compiler.IErrorReporter import prog8.compiler.target.ICompilationTarget +import kotlin.math.abs + + +fun RangeExpr.size(encoding: IStringEncoding): Int? { + val fromLv = (from as? NumericLiteralValue) + val toLv = (to as? NumericLiteralValue) + if(fromLv==null || toLv==null) + return null + return toConstantIntegerRange(encoding)?.count() +} + +fun RangeExpr.toConstantIntegerRange(encoding: IStringEncoding): IntProgression? { + val fromVal: Int + val toVal: Int + val fromString = from as? StringLiteralValue + val toString = to as? StringLiteralValue + if(fromString!=null && toString!=null ) { + // string range -> int range over character values + fromVal = encoding.encodeString(fromString.value, fromString.altEncoding)[0].toInt() + toVal = encoding.encodeString(toString.value, fromString.altEncoding)[0].toInt() + } else { + val fromLv = from as? NumericLiteralValue + val toLv = to as? NumericLiteralValue + if(fromLv==null || toLv==null) + return null // non-constant range + // integer range + fromVal = fromLv.number.toInt() + toVal = toLv.number.toInt() + } + val stepVal = (step as? NumericLiteralValue)?.number?.toInt() ?: 1 + return makeRange(fromVal, toVal, stepVal) +} + +private fun makeRange(fromVal: Int, toVal: Int, stepVal: Int): IntProgression { + return when { + fromVal <= toVal -> when { + stepVal <= 0 -> IntRange.EMPTY + stepVal == 1 -> fromVal..toVal + else -> fromVal..toVal step stepVal + } + else -> when { + stepVal >= 0 -> IntRange.EMPTY + stepVal == -1 -> fromVal downTo toVal + else -> fromVal downTo toVal step abs(stepVal) + } + } +} internal fun Program.checkValid(compilerOptions: CompilationOptions, errors: IErrorReporter, compTarget: ICompilationTarget) { diff --git a/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt b/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt index 1aa1a8bff..226c4e7b8 100644 --- a/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt +++ b/compiler/src/prog8/compiler/target/cpu6502/codegen/ForLoopsAsmGen.kt @@ -9,6 +9,7 @@ import prog8.ast.expressions.RangeExpr import prog8.ast.statements.ForLoop import prog8.ast.toHex import prog8.compiler.AssemblyError +import prog8.compiler.astprocessing.toConstantIntegerRange import kotlin.math.absoluteValue internal class ForLoopsAsmGen(private val program: Program, private val asmgen: AsmGen) { diff --git a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt index f45bfd4a1..cfab08140 100644 --- a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt +++ b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt @@ -9,6 +9,8 @@ import prog8.ast.statements.* import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstModification import prog8.compiler.IErrorReporter +import prog8.compiler.astprocessing.size +import prog8.compiler.astprocessing.toConstantIntegerRange import prog8.compiler.target.ICompilationTarget // Fix up the literal value's type to match that of the vardecl diff --git a/compiler/src/prog8/optimizer/StatementOptimizer.kt b/compiler/src/prog8/optimizer/StatementOptimizer.kt index af7455e22..5688cb8cf 100644 --- a/compiler/src/prog8/optimizer/StatementOptimizer.kt +++ b/compiler/src/prog8/optimizer/StatementOptimizer.kt @@ -11,6 +11,7 @@ import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstModification import prog8.ast.walk.IAstVisitor import prog8.compiler.IErrorReporter +import prog8.compiler.astprocessing.size import prog8.compiler.target.ICompilationTarget import kotlin.math.floor diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index 18e7bd16f..c6c36cc78 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -14,6 +14,8 @@ import prog8.ast.expressions.* import prog8.ast.statements.ForLoop import prog8.ast.statements.Subroutine import prog8.ast.statements.VarDecl +import prog8.compiler.astprocessing.size +import prog8.compiler.astprocessing.toConstantIntegerRange import prog8.compiler.target.C64Target import prog8.compiler.target.Cx16Target diff --git a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt index 48205ff30..de7994e4c 100644 --- a/compilerAst/src/prog8/ast/expressions/AstExpressions.kt +++ b/compilerAst/src/prog8/ast/expressions/AstExpressions.kt @@ -7,7 +7,6 @@ import prog8.ast.statements.* import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstVisitor import java.util.* -import kotlin.math.abs val associativeOperators = setOf("+", "*", "&", "|", "^", "or", "and", "xor", "==", "!=") @@ -719,51 +718,8 @@ class RangeExpr(var from: Expression, return "RangeExpr(from $from, to $to, step $step, pos=$position)" } - fun size(encoding: IStringEncoding): Int? { - val fromLv = (from as? NumericLiteralValue) - val toLv = (to as? NumericLiteralValue) - if(fromLv==null || toLv==null) - return null - return toConstantIntegerRange(encoding)?.count() - } - - fun toConstantIntegerRange(encoding: IStringEncoding): IntProgression? { - val fromVal: Int - val toVal: Int - val fromString = from as? StringLiteralValue - val toString = to as? StringLiteralValue - if(fromString!=null && toString!=null ) { - // string range -> int range over character values - fromVal = encoding.encodeString(fromString.value, fromString.altEncoding)[0].toInt() - toVal = encoding.encodeString(toString.value, fromString.altEncoding)[0].toInt() - } else { - val fromLv = from as? NumericLiteralValue - val toLv = to as? NumericLiteralValue - if(fromLv==null || toLv==null) - return null // non-constant range - // integer range - fromVal = fromLv.number.toInt() - toVal = toLv.number.toInt() - } - val stepVal = (step as? NumericLiteralValue)?.number?.toInt() ?: 1 - return makeRange(fromVal, toVal, stepVal) - } } -internal fun makeRange(fromVal: Int, toVal: Int, stepVal: Int): IntProgression { - return when { - fromVal <= toVal -> when { - stepVal <= 0 -> IntRange.EMPTY - stepVal == 1 -> fromVal..toVal - else -> fromVal..toVal step stepVal - } - else -> when { - stepVal >= 0 -> IntRange.EMPTY - stepVal == -1 -> fromVal downTo toVal - else -> fromVal downTo toVal step abs(stepVal) - } - } -} data class IdentifierReference(val nameInSource: List, override val position: Position) : Expression(), IAssignable { override lateinit var parent: Node diff --git a/compilerAst/src/prog8/parser/PetsciiEncoding.kt b/compilerAst/src/prog8/parser/PetsciiEncoding.kt index 58f752834..9817003bd 100644 --- a/compilerAst/src/prog8/parser/PetsciiEncoding.kt +++ b/compilerAst/src/prog8/parser/PetsciiEncoding.kt @@ -5,7 +5,7 @@ import java.io.CharConversionException /** - * TODO: remove once [IStringEncoding] has been to compiler module + * TODO: remove once [IStringEncoding] has been moved to compiler module */ object PetsciiEncoding : IStringEncoding { override fun encodeString(str: String, altEncoding: Boolean) = diff --git a/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt b/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt index 674c3d77e..57f48871e 100644 --- a/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt +++ b/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt @@ -3,7 +3,7 @@ package prog8.parser import prog8.ast.IStringEncoding /** - * TODO: remove once [IStringEncoding] has been to compiler module + * TODO: remove once [IStringEncoding] has been moved to compiler module */ object ThrowTodoEncoding: IStringEncoding { override fun encodeString(str: String, altEncoding: Boolean): List { From de92740e8703a45753156e3ffe48d130e40879de Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 17 Jul 2021 21:29:01 +0200 Subject: [PATCH 39/68] * simple refactoring: move IStringEncoding, IMemSizer and IBuiltinFunctions to files of their own, also ext method Number.toHex to file compilerAst/src/prog8/ast/Extensions.kt. Moved the other interfaces that are indeed closely related to Node to the top of AstToplevel.kt. *This is really ONLY moving text around*, so it's easier to find things. Nothing else. --- compilerAst/src/prog8/ast/AstToplevel.kt | 116 +++++++----------- compilerAst/src/prog8/ast/Extensions.kt | 19 +++ .../src/prog8/ast/IBuiltinFunctions.kt | 13 ++ compilerAst/src/prog8/ast/IMemSizer.kt | 7 ++ compilerAst/src/prog8/ast/IStringEncoding.kt | 6 + 5 files changed, 87 insertions(+), 74 deletions(-) create mode 100644 compilerAst/src/prog8/ast/Extensions.kt create mode 100644 compilerAst/src/prog8/ast/IBuiltinFunctions.kt create mode 100644 compilerAst/src/prog8/ast/IMemSizer.kt create mode 100644 compilerAst/src/prog8/ast/IStringEncoding.kt diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index dfbf16310..4c67f054c 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -6,54 +6,12 @@ import prog8.ast.statements.* import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstVisitor import prog8.parser.SourceCode -import kotlin.math.abs const val internedStringsModuleName = "prog8_interned_strings" -interface IStringEncoding { - fun encodeString(str: String, altEncoding: Boolean): List - fun decodeString(bytes: List, altEncoding: Boolean): String -} -interface Node { - val position: Position - var parent: Node // will be linked correctly later (late init) - fun linkParents(parent: Node) - - fun definingModule(): Module { - if(this is Module) - return this - return findParentNode(this)!! - } - - fun definingSubroutine(): Subroutine? = findParentNode(this) - - fun definingScope(): INameScope { - val scope = findParentNode(this) - if(scope!=null) { - return scope - } - if(this is Label && this.name.startsWith("builtin::")) { - return BuiltinFunctionScopePlaceholder - } - if(this is GlobalNamespace) - return this - throw FatalAstException("scope missing from $this") - } - - fun definingBlock(): Block { - if(this is Block) - return this - return findParentNode(this)!! - } - - fun containingStatement(): Statement { - if(this is Statement) - return this - return findParentNode(this)!! - } - - fun replaceChildNode(node: Node, replacement: Node) +interface IAssignable { + // just a tag for now } interface IFunctionCall { @@ -61,7 +19,6 @@ interface IFunctionCall { var args: MutableList } - interface INameScope { val name: String val position: Position @@ -227,25 +184,51 @@ interface INameScope { } } -interface IAssignable { - // just a tag for now + +interface Node { + val position: Position + var parent: Node // will be linked correctly later (late init) + fun linkParents(parent: Node) + + fun definingModule(): Module { + if(this is Module) + return this + return findParentNode(this)!! + } + + fun definingSubroutine(): Subroutine? = findParentNode(this) + + fun definingScope(): INameScope { + val scope = findParentNode(this) + if(scope!=null) { + return scope + } + if(this is Label && this.name.startsWith("builtin::")) { + return BuiltinFunctionScopePlaceholder + } + if(this is GlobalNamespace) + return this + throw FatalAstException("scope missing from $this") + } + + fun definingBlock(): Block { + if(this is Block) + return this + return findParentNode(this)!! + } + + fun containingStatement(): Statement { + if(this is Statement) + return this + return findParentNode(this)!! + } + + fun replaceChildNode(node: Node, replacement: Node) } -interface IMemSizer { - fun memorySize(dt: DataType): Int -} - -interface IBuiltinFunctions { - val names: Set - val purefunctionNames: Set - fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? - fun returnType(name: String, args: MutableList): InferredTypes.InferredType -} /*********** Everything starts from here, the Program; zero or more modules *************/ - - class Program(val name: String, val modules: MutableList, val builtinFunctions: IBuiltinFunctions, @@ -419,18 +402,3 @@ object BuiltinFunctionScopePlaceholder : INameScope { } -fun Number.toHex(): String { - // 0..15 -> "0".."15" - // 16..255 -> "$10".."$ff" - // 256..65536 -> "$0100".."$ffff" - // negative values are prefixed with '-'. - val integer = this.toInt() - if(integer<0) - return '-' + abs(integer).toHex() - return when (integer) { - in 0 until 16 -> integer.toString() - in 0 until 0x100 -> "$"+integer.toString(16).padStart(2,'0') - in 0 until 0x10000 -> "$"+integer.toString(16).padStart(4,'0') - else -> throw IllegalArgumentException("number too large for 16 bits $this") - } -} diff --git a/compilerAst/src/prog8/ast/Extensions.kt b/compilerAst/src/prog8/ast/Extensions.kt new file mode 100644 index 000000000..5d6bf84eb --- /dev/null +++ b/compilerAst/src/prog8/ast/Extensions.kt @@ -0,0 +1,19 @@ +package prog8.ast + +import kotlin.math.abs + +fun Number.toHex(): String { + // 0..15 -> "0".."15" + // 16..255 -> "$10".."$ff" + // 256..65536 -> "$0100".."$ffff" + // negative values are prefixed with '-'. + val integer = this.toInt() + if(integer<0) + return '-' + abs(integer).toHex() + return when (integer) { + in 0 until 16 -> integer.toString() + in 0 until 0x100 -> "$"+integer.toString(16).padStart(2,'0') + in 0 until 0x10000 -> "$"+integer.toString(16).padStart(4,'0') + else -> throw IllegalArgumentException("number too large for 16 bits $this") + } +} \ No newline at end of file diff --git a/compilerAst/src/prog8/ast/IBuiltinFunctions.kt b/compilerAst/src/prog8/ast/IBuiltinFunctions.kt new file mode 100644 index 000000000..b9d19cf58 --- /dev/null +++ b/compilerAst/src/prog8/ast/IBuiltinFunctions.kt @@ -0,0 +1,13 @@ +package prog8.ast + +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue + +interface IBuiltinFunctions { + val names: Set + val purefunctionNames: Set + fun constValue(name: String, args: List, position: Position, memsizer: IMemSizer): NumericLiteralValue? + fun returnType(name: String, args: MutableList): InferredTypes.InferredType +} \ No newline at end of file diff --git a/compilerAst/src/prog8/ast/IMemSizer.kt b/compilerAst/src/prog8/ast/IMemSizer.kt new file mode 100644 index 000000000..9ba6767e3 --- /dev/null +++ b/compilerAst/src/prog8/ast/IMemSizer.kt @@ -0,0 +1,7 @@ +package prog8.ast + +import prog8.ast.base.DataType + +interface IMemSizer { + fun memorySize(dt: DataType): Int +} \ No newline at end of file diff --git a/compilerAst/src/prog8/ast/IStringEncoding.kt b/compilerAst/src/prog8/ast/IStringEncoding.kt new file mode 100644 index 000000000..8296d2988 --- /dev/null +++ b/compilerAst/src/prog8/ast/IStringEncoding.kt @@ -0,0 +1,6 @@ +package prog8.ast + +interface IStringEncoding { + fun encodeString(str: String, altEncoding: Boolean): List + fun decodeString(bytes: List, altEncoding: Boolean): String +} \ No newline at end of file From db76c8d7f4c6cfda20aa716800131f47c67bcb81 Mon Sep 17 00:00:00 2001 From: meisl Date: Tue, 13 Jul 2021 09:51:34 +0200 Subject: [PATCH 40/68] -/* remove IStringEncoding as param in compilerAst, and all other uses that were only because of that. For good measure we also turn on *all* compiler tests with examples (they do take some time). Note that the total *mentions* of IStringEncoding in the entire project went down from ~50 to 6, only 3 of which are *actual uses* (the others are 2 imports and 1 supertype ref in ICompilationTarget : IStringEncoding)! --- compiler/src/prog8/compiler/Compiler.kt | 14 +- .../optimizer/ConstantFoldingOptimizer.kt | 3 +- compiler/src/prog8/optimizer/Extensions.kt | 2 +- compiler/test/Helpers.kt | 11 - compiler/test/TestCompilerOnExamples.kt | 2 +- .../src/prog8/ast/antlr/Antlr2Kotlin.kt | 202 ++- compilerAst/src/prog8/parser/ModuleParsing.kt | 2 - compilerAst/src/prog8/parser/Petscii.kt | 1171 ----------------- .../src/prog8/parser/PetsciiEncoding.kt | 24 - compilerAst/src/prog8/parser/Prog8Parser.kt | 2 +- .../src/prog8/parser/ThrowTodoEncoding.kt | 16 - compilerAst/test/Helpers.kt | 11 - compilerAst/test/TestModuleImporter.kt | 14 +- 13 files changed, 115 insertions(+), 1359 deletions(-) delete mode 100644 compilerAst/src/prog8/parser/Petscii.kt delete mode 100644 compilerAst/src/prog8/parser/PetsciiEncoding.kt delete mode 100644 compilerAst/src/prog8/parser/ThrowTodoEncoding.kt diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index a32d2b5fb..2fb3f03eb 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -1,6 +1,9 @@ package prog8.compiler -import prog8.ast.* +import prog8.ast.AstToSourceCode +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.Program import prog8.ast.base.AstException import prog8.ast.base.Position import prog8.ast.expressions.Expression @@ -171,13 +174,12 @@ private fun parseImports(filepath: Path, errors: IErrorReporter, compTarget: ICompilationTarget, libdirs: List): Triple> { - val compilationTargetName = compTarget.name - println("Compiler target: $compilationTargetName. Parsing...") + println("Compiler target: ${compTarget.name}. Parsing...") val bf = BuiltinFunctionsFacade(BuiltinFunctions) val programAst = Program(moduleName(filepath.fileName), mutableListOf(), bf, compTarget) bf.program = programAst - val importer = ModuleImporter(programAst, compTarget, compilationTargetName, libdirs) + val importer = ModuleImporter(programAst, compTarget.name, libdirs) importer.importModule(filepath) errors.report() @@ -190,7 +192,7 @@ private fun parseImports(filepath: Path, throw ParsingFailedError("${programAst.modules.first().position} BASIC launcher requires output type PRG.") // depending on the machine and compiler options we may have to include some libraries - for(lib in compTarget.machine.importLibs(compilerOptions, compilationTargetName)) + for(lib in compTarget.machine.importLibs(compilerOptions, compTarget.name)) importer.importLibraryModule(lib) // always import prog8_lib and math @@ -266,7 +268,7 @@ private fun processAst(programAst: Program, errors: IErrorReporter, compilerOpti programAst.checkIdentifiers(errors, compilerOptions) errors.report() // TODO: turning char literals into UBYTEs via an encoding should really happen in code gen - but for that we'd need DataType.CHAR - programAst.charLiteralsToUByteLiterals(errors, compilerOptions.compTarget as IStringEncoding) + programAst.charLiteralsToUByteLiterals(errors, compilerOptions.compTarget) errors.report() programAst.constantFold(errors, compilerOptions.compTarget) errors.report() diff --git a/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt b/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt index abe9ce509..9b6fc280a 100644 --- a/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt +++ b/compiler/src/prog8/optimizer/ConstantFoldingOptimizer.kt @@ -9,11 +9,10 @@ import prog8.ast.statements.ForLoop import prog8.ast.statements.VarDecl import prog8.ast.walk.AstWalker import prog8.ast.walk.IAstModification -import prog8.compiler.target.ICompilationTarget import kotlin.math.pow -internal class ConstantFoldingOptimizer(private val program: Program, private val compTarget: ICompilationTarget) : AstWalker() { +internal class ConstantFoldingOptimizer(private val program: Program) : AstWalker() { override fun before(memread: DirectMemoryRead, parent: Node): Iterable { // @( &thing ) --> thing diff --git a/compiler/src/prog8/optimizer/Extensions.kt b/compiler/src/prog8/optimizer/Extensions.kt index 89bd8910f..b69be66a7 100644 --- a/compiler/src/prog8/optimizer/Extensions.kt +++ b/compiler/src/prog8/optimizer/Extensions.kt @@ -21,7 +21,7 @@ internal fun Program.constantFold(errors: IErrorReporter, compTarget: ICompilati if(errors.noErrors()) { valuetypefixer.applyModifications() - val optimizer = ConstantFoldingOptimizer(this, compTarget) + val optimizer = ConstantFoldingOptimizer(this) optimizer.visit(this) while (errors.noErrors() && optimizer.applyModifications() > 0) { optimizer.visit(this) diff --git a/compiler/test/Helpers.kt b/compiler/test/Helpers.kt index 0ea41bf05..48f745b2d 100644 --- a/compiler/test/Helpers.kt +++ b/compiler/test/Helpers.kt @@ -6,7 +6,6 @@ import java.nio.file.Path import prog8.ast.IBuiltinFunctions import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.Expression @@ -127,16 +126,6 @@ fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: }.toList() -val DummyEncoding = object : IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - throw Exception("just a dummy - should not be called") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - throw Exception("just a dummy - should not be called") - } -} - val DummyFunctions = object : IBuiltinFunctions { override val names: Set = emptySet() override val purefunctionNames: Set = emptySet() diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index 361d5f174..a98e2bade 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -54,7 +54,7 @@ class TestCompilerOnExamples { } @TestFactory - @Disabled +// @Disabled fun bothCx16AndC64() = mapCombinations( dim1 = listOf( "animals", diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index ed6a10c2c..2a695dee6 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -2,26 +2,16 @@ package prog8.ast.antlr import org.antlr.v4.runtime.ParserRuleContext import org.antlr.v4.runtime.tree.TerminalNode -import prog8.ast.IStringEncoding -import prog8.ast.Module import prog8.ast.base.* import prog8.ast.expressions.* import prog8.ast.statements.* import prog8.parser.Prog8ANTLRParser -import prog8.parser.SourceCode /***************** Antlr Extension methods to create AST ****************/ private data class NumericLiteral(val number: Number, val datatype: DataType) -internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: SourceCode, encoding: IStringEncoding) : Module { - val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name - val directives = this.directive().map { it.toAst() } - val blocks = this.block().map { it.toAst(isInLibrary = source.isFromResources, encoding) } - return Module(nameWithoutSuffix, (directives + blocks).toMutableList(), toPosition(), source) -} - private fun ParserRuleContext.toPosition() : Position { /* val customTokensource = this.start.tokenSource as? CustomLexer @@ -38,11 +28,11 @@ private fun ParserRuleContext.toPosition() : Position { return Position(filename, start.line, start.charPositionInLine, stop.charPositionInLine + stop.text.length) } -internal fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: IStringEncoding) : Block { +internal fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean) : Block { val blockstatements = block_statement().map { when { - it.variabledeclaration()!=null -> it.variabledeclaration().toAst(encoding) - it.subroutinedeclaration()!=null -> it.subroutinedeclaration().toAst(encoding) + it.variabledeclaration()!=null -> it.variabledeclaration().toAst() + it.subroutinedeclaration()!=null -> it.subroutinedeclaration().toAst() it.directive()!=null -> it.directive().toAst() it.inlineasm()!=null -> it.inlineasm().toAst() it.labeldef()!=null -> it.labeldef().toAst() @@ -52,11 +42,11 @@ internal fun Prog8ANTLRParser.BlockContext.toAst(isInLibrary: Boolean, encoding: return Block(identifier().text, integerliteral()?.toAst()?.number?.toInt(), blockstatements.toMutableList(), isInLibrary, toPosition()) } -private fun Prog8ANTLRParser.Statement_blockContext.toAst(encoding: IStringEncoding): MutableList = - statement().asSequence().map { it.toAst(encoding) }.toMutableList() +private fun Prog8ANTLRParser.Statement_blockContext.toAst(): MutableList = + statement().asSequence().map { it.toAst() }.toMutableList() -private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringEncoding) : Statement { - vardecl()?.let { return it.toAst(encoding) } +private fun Prog8ANTLRParser.VariabledeclarationContext.toAst() : Statement { + vardecl()?.let { return it.toAst() } varinitializer()?.let { val vd = it.vardecl() @@ -64,9 +54,9 @@ private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringE VarDeclType.VAR, vd.datatype()?.toAst() ?: DataType.UNDEFINED, if (vd.ZEROPAGE() != null) ZeropageWish.PREFER_ZEROPAGE else ZeropageWish.DONTCARE, - vd.arrayindex()?.toAst(encoding), + vd.arrayindex()?.toAst(), vd.varname.text, - it.expression().toAst(encoding), + it.expression().toAst(), vd.ARRAYSIG() != null || vd.arrayindex() != null, false, vd.SHARED()!=null, @@ -81,9 +71,9 @@ private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringE VarDeclType.CONST, vd.datatype()?.toAst() ?: DataType.UNDEFINED, if (vd.ZEROPAGE() != null) ZeropageWish.PREFER_ZEROPAGE else ZeropageWish.DONTCARE, - vd.arrayindex()?.toAst(encoding), + vd.arrayindex()?.toAst(), vd.varname.text, - cvarinit.expression().toAst(encoding), + cvarinit.expression().toAst(), vd.ARRAYSIG() != null || vd.arrayindex() != null, false, vd.SHARED() != null, @@ -98,9 +88,9 @@ private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringE VarDeclType.MEMORY, vd.datatype()?.toAst() ?: DataType.UNDEFINED, if (vd.ZEROPAGE() != null) ZeropageWish.PREFER_ZEROPAGE else ZeropageWish.DONTCARE, - vd.arrayindex()?.toAst(encoding), + vd.arrayindex()?.toAst(), vd.varname.text, - mvarinit.expression().toAst(encoding), + mvarinit.expression().toAst(), vd.ARRAYSIG() != null || vd.arrayindex() != null, false, vd.SHARED()!=null, @@ -111,33 +101,33 @@ private fun Prog8ANTLRParser.VariabledeclarationContext.toAst(encoding: IStringE throw FatalAstException("weird variable decl $this") } -private fun Prog8ANTLRParser.SubroutinedeclarationContext.toAst(encoding: IStringEncoding) : Subroutine { +private fun Prog8ANTLRParser.SubroutinedeclarationContext.toAst() : Subroutine { return when { - subroutine()!=null -> subroutine().toAst(encoding) - asmsubroutine()!=null -> asmsubroutine().toAst(encoding) + subroutine()!=null -> subroutine().toAst() + asmsubroutine()!=null -> asmsubroutine().toAst() romsubroutine()!=null -> romsubroutine().toAst() else -> throw FatalAstException("weird subroutine decl $this") } } -private fun Prog8ANTLRParser.StatementContext.toAst(encoding: IStringEncoding) : Statement { - val vardecl = variabledeclaration()?.toAst(encoding) +private fun Prog8ANTLRParser.StatementContext.toAst() : Statement { + val vardecl = variabledeclaration()?.toAst() if(vardecl!=null) return vardecl assignment()?.let { - return Assignment(it.assign_target().toAst(encoding), it.expression().toAst(encoding), it.toPosition()) + return Assignment(it.assign_target().toAst(), it.expression().toAst(), it.toPosition()) } augassignment()?.let { // replace A += X with A = A + X - val target = it.assign_target().toAst(encoding) + val target = it.assign_target().toAst() val oper = it.operator.text.substringBefore('=') - val expression = BinaryExpression(target.toExpression(), oper, it.expression().toAst(encoding), it.expression().toPosition()) - return Assignment(it.assign_target().toAst(encoding), expression, it.toPosition()) + val expression = BinaryExpression(target.toExpression(), oper, it.expression().toAst(), it.expression().toPosition()) + return Assignment(it.assign_target().toAst(), expression, it.toPosition()) } postincrdecr()?.let { - return PostIncrDecr(it.assign_target().toAst(encoding), it.operator.text, it.toPosition()) + return PostIncrDecr(it.assign_target().toAst(), it.operator.text, it.toPosition()) } val directive = directive()?.toAst() @@ -149,49 +139,49 @@ private fun Prog8ANTLRParser.StatementContext.toAst(encoding: IStringEncoding) : val jump = unconditionaljump()?.toAst() if(jump!=null) return jump - val fcall = functioncall_stmt()?.toAst(encoding) + val fcall = functioncall_stmt()?.toAst() if(fcall!=null) return fcall - val ifstmt = if_stmt()?.toAst(encoding) + val ifstmt = if_stmt()?.toAst() if(ifstmt!=null) return ifstmt - val returnstmt = returnstmt()?.toAst(encoding) + val returnstmt = returnstmt()?.toAst() if(returnstmt!=null) return returnstmt - val subroutine = subroutinedeclaration()?.toAst(encoding) + val subroutine = subroutinedeclaration()?.toAst() if(subroutine!=null) return subroutine val asm = inlineasm()?.toAst() if(asm!=null) return asm - val branchstmt = branch_stmt()?.toAst(encoding) + val branchstmt = branch_stmt()?.toAst() if(branchstmt!=null) return branchstmt - val forloop = forloop()?.toAst(encoding) + val forloop = forloop()?.toAst() if(forloop!=null) return forloop - val untilloop = untilloop()?.toAst(encoding) + val untilloop = untilloop()?.toAst() if(untilloop!=null) return untilloop - val whileloop = whileloop()?.toAst(encoding) + val whileloop = whileloop()?.toAst() if(whileloop!=null) return whileloop - val repeatloop = repeatloop()?.toAst(encoding) + val repeatloop = repeatloop()?.toAst() if(repeatloop!=null) return repeatloop val breakstmt = breakstmt()?.toAst() if(breakstmt!=null) return breakstmt - val whenstmt = whenstmt()?.toAst(encoding) + val whenstmt = whenstmt()?.toAst() if(whenstmt!=null) return whenstmt throw FatalAstException("unprocessed source text (are we missing ast conversion rules for parser elements?): $text") } -private fun Prog8ANTLRParser.AsmsubroutineContext.toAst(encoding: IStringEncoding): Subroutine { +private fun Prog8ANTLRParser.AsmsubroutineContext.toAst(): Subroutine { val inline = this.inline()!=null val subdecl = asmsub_decl().toAst() - val statements = statement_block()?.toAst(encoding) ?: mutableListOf() + val statements = statement_block()?.toAst() ?: mutableListOf() return Subroutine(subdecl.name, subdecl.parameters, subdecl.returntypes, subdecl.asmParameterRegisters, subdecl.asmReturnvaluesRegisters, subdecl.asmClobbers, null, true, inline, statements, toPosition()) @@ -272,28 +262,28 @@ private fun Prog8ANTLRParser.Asmsub_paramsContext.toAst(): List AssignTarget(identifier.toAst(), null, null, toPosition()) - arrayindexed()!=null -> AssignTarget(null, arrayindexed().toAst(encoding), null, toPosition()) - directmemory()!=null -> AssignTarget(null, null, DirectMemoryWrite(directmemory().expression().toAst(encoding), toPosition()), toPosition()) + arrayindexed()!=null -> AssignTarget(null, arrayindexed().toAst(), null, toPosition()) + directmemory()!=null -> AssignTarget(null, null, DirectMemoryWrite(directmemory().expression().toAst(), toPosition()), toPosition()) else -> AssignTarget(scoped_identifier()?.toAst(), null, null, toPosition()) } } @@ -345,8 +335,8 @@ private fun Prog8ANTLRParser.ClobberContext.toAst() : Set { private fun Prog8ANTLRParser.DatatypeContext.toAst() = DataType.valueOf(text.uppercase()) -private fun Prog8ANTLRParser.ArrayindexContext.toAst(encoding: IStringEncoding) : ArrayIndex = - ArrayIndex(expression().toAst(encoding), toPosition()) +private fun Prog8ANTLRParser.ArrayindexContext.toAst() : ArrayIndex = + ArrayIndex(expression().toAst(), toPosition()) internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = Directive(directivename.text, directivearg().map { it.toAst() }, toPosition()) @@ -354,7 +344,7 @@ internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { val str = stringliteral() if(str?.ALT_STRING_ENCODING() != null) - throw AstException("${toPosition()} can't use alternate string encodings for directive arguments") + throw AstException("${toPosition()} can't use alternate string s for directive arguments") return DirectiveArg(stringliteral()?.text, identifier()?.text, integerliteral()?.toAst()?.number?.toInt(), toPosition()) } @@ -410,7 +400,7 @@ private fun Prog8ANTLRParser.IntegerliteralContext.toAst(): NumericLiteral { } } -private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) : Expression { +private fun Prog8ANTLRParser.ExpressionContext.toAst() : Expression { val litval = literalvalue() if(litval!=null) { @@ -433,7 +423,7 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) litval.stringliteral()!=null -> litval.stringliteral().toAst() litval.charliteral()!=null -> litval.charliteral().toAst() litval.arrayliteral()!=null -> { - val array = litval.arrayliteral().toAst(encoding) + val array = litval.arrayliteral().toAst() // the actual type of the arraysize can not yet be determined here (missing namespace & heap) // the ConstantFold takes care of that and converts the type if needed. ArrayLiteralValue(InferredTypes.InferredType.unknown(), array, position = litval.toPosition()) @@ -447,31 +437,31 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst(encoding: IStringEncoding) return scoped_identifier().toAst() if(bop!=null) - return BinaryExpression(left.toAst(encoding), bop.text, right.toAst(encoding), toPosition()) + return BinaryExpression(left.toAst(), bop.text, right.toAst(), toPosition()) if(prefix!=null) - return PrefixExpression(prefix.text, expression(0).toAst(encoding), toPosition()) + return PrefixExpression(prefix.text, expression(0).toAst(), toPosition()) - val funcall = functioncall()?.toAst(encoding) + val funcall = functioncall()?.toAst() if(funcall!=null) return funcall if (rangefrom!=null && rangeto!=null) { val defaultstep = if(rto.text == "to") 1 else -1 - val step = rangestep?.toAst(encoding) ?: NumericLiteralValue(DataType.UBYTE, defaultstep, toPosition()) - return RangeExpr(rangefrom.toAst(encoding), rangeto.toAst(encoding), step, toPosition()) + val step = rangestep?.toAst() ?: NumericLiteralValue(DataType.UBYTE, defaultstep, toPosition()) + return RangeExpr(rangefrom.toAst(), rangeto.toAst(), step, toPosition()) } if(childCount==3 && children[0].text=="(" && children[2].text==")") - return expression(0).toAst(encoding) // expression within ( ) + return expression(0).toAst() // expression within ( ) if(arrayindexed()!=null) - return arrayindexed().toAst(encoding) + return arrayindexed().toAst() if(typecast()!=null) - return TypecastExpression(expression(0).toAst(encoding), typecast().datatype().toAst(), false, toPosition()) + return TypecastExpression(expression(0).toAst(), typecast().datatype().toAst(), false, toPosition()) if(directmemory()!=null) - return DirectMemoryRead(directmemory().expression().toAst(encoding), toPosition()) + return DirectMemoryRead(directmemory().expression().toAst(), toPosition()) if(addressof()!=null) return AddressOf(addressof().scoped_identifier().toAst(), toPosition()) @@ -485,13 +475,13 @@ private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral = private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue = StringLiteralValue(unescape(this.STRING().text, toPosition()), ALT_STRING_ENCODING()!=null, toPosition()) -private fun Prog8ANTLRParser.ArrayindexedContext.toAst(encoding: IStringEncoding): ArrayIndexedExpression { +private fun Prog8ANTLRParser.ArrayindexedContext.toAst(): ArrayIndexedExpression { return ArrayIndexedExpression(scoped_identifier().toAst(), - arrayindex().toAst(encoding), + arrayindex().toAst(), toPosition()) } -private fun Prog8ANTLRParser.Expression_listContext.toAst(encoding: IStringEncoding) = expression().map{ it.toAst(encoding) } +private fun Prog8ANTLRParser.Expression_listContext.toAst() = expression().map{ it.toAst() } private fun Prog8ANTLRParser.IdentifierContext.toAst() : IdentifierReference = IdentifierReference(listOf(text), toPosition()) @@ -507,27 +497,27 @@ private fun Prog8ANTLRParser.BooleanliteralContext.toAst() = when(text) { else -> throw FatalAstException(text) } -private fun Prog8ANTLRParser.ArrayliteralContext.toAst(encoding: IStringEncoding) : Array = - expression().map { it.toAst(encoding) }.toTypedArray() +private fun Prog8ANTLRParser.ArrayliteralContext.toAst() : Array = + expression().map { it.toAst() }.toTypedArray() -private fun Prog8ANTLRParser.If_stmtContext.toAst(encoding: IStringEncoding): IfStatement { - val condition = expression().toAst(encoding) - val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) - val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() +private fun Prog8ANTLRParser.If_stmtContext.toAst(): IfStatement { + val condition = expression().toAst() + val trueStatements = statement_block()?.toAst() ?: mutableListOf(statement().toAst()) + val elseStatements = else_part()?.toAst() ?: mutableListOf() val trueScope = AnonymousScope(trueStatements, statement_block()?.toPosition() ?: statement().toPosition()) val elseScope = AnonymousScope(elseStatements, else_part()?.toPosition() ?: toPosition()) return IfStatement(condition, trueScope, elseScope, toPosition()) } -private fun Prog8ANTLRParser.Else_partContext.toAst(encoding: IStringEncoding): MutableList { - return statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) +private fun Prog8ANTLRParser.Else_partContext.toAst(): MutableList { + return statement_block()?.toAst() ?: mutableListOf(statement().toAst()) } -private fun Prog8ANTLRParser.Branch_stmtContext.toAst(encoding: IStringEncoding): BranchStatement { +private fun Prog8ANTLRParser.Branch_stmtContext.toAst(): BranchStatement { val branchcondition = branchcondition().toAst() - val trueStatements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) - val elseStatements = else_part()?.toAst(encoding) ?: mutableListOf() + val trueStatements = statement_block()?.toAst() ?: mutableListOf(statement().toAst()) + val elseStatements = else_part()?.toAst() ?: mutableListOf() val trueScope = AnonymousScope(trueStatements, statement_block()?.toPosition() ?: statement().toPosition()) val elseScope = AnonymousScope(elseStatements, else_part()?.toPosition() ?: toPosition()) @@ -538,65 +528,65 @@ private fun Prog8ANTLRParser.BranchconditionContext.toAst() = BranchCondition.va text.substringAfter('_').uppercase() ) -private fun Prog8ANTLRParser.ForloopContext.toAst(encoding: IStringEncoding): ForLoop { +private fun Prog8ANTLRParser.ForloopContext.toAst(): ForLoop { val loopvar = identifier().toAst() - val iterable = expression()!!.toAst(encoding) + val iterable = expression()!!.toAst() val scope = if(statement()!=null) - AnonymousScope(mutableListOf(statement().toAst(encoding)), statement().toPosition()) + AnonymousScope(mutableListOf(statement().toAst()), statement().toPosition()) else - AnonymousScope(statement_block().toAst(encoding), statement_block().toPosition()) + AnonymousScope(statement_block().toAst(), statement_block().toPosition()) return ForLoop(loopvar, iterable, scope, toPosition()) } private fun Prog8ANTLRParser.BreakstmtContext.toAst() = Break(toPosition()) -private fun Prog8ANTLRParser.WhileloopContext.toAst(encoding: IStringEncoding): WhileLoop { - val condition = expression().toAst(encoding) - val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) +private fun Prog8ANTLRParser.WhileloopContext.toAst(): WhileLoop { + val condition = expression().toAst() + val statements = statement_block()?.toAst() ?: mutableListOf(statement().toAst()) val scope = AnonymousScope(statements, statement_block()?.toPosition() ?: statement().toPosition()) return WhileLoop(condition, scope, toPosition()) } -private fun Prog8ANTLRParser.RepeatloopContext.toAst(encoding: IStringEncoding): RepeatLoop { - val iterations = expression()?.toAst(encoding) - val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) +private fun Prog8ANTLRParser.RepeatloopContext.toAst(): RepeatLoop { + val iterations = expression()?.toAst() + val statements = statement_block()?.toAst() ?: mutableListOf(statement().toAst()) val scope = AnonymousScope(statements, statement_block()?.toPosition() ?: statement().toPosition()) return RepeatLoop(iterations, scope, toPosition()) } -private fun Prog8ANTLRParser.UntilloopContext.toAst(encoding: IStringEncoding): UntilLoop { - val untilCondition = expression().toAst(encoding) - val statements = statement_block()?.toAst(encoding) ?: mutableListOf(statement().toAst(encoding)) +private fun Prog8ANTLRParser.UntilloopContext.toAst(): UntilLoop { + val untilCondition = expression().toAst() + val statements = statement_block()?.toAst() ?: mutableListOf(statement().toAst()) val scope = AnonymousScope(statements, statement_block()?.toPosition() ?: statement().toPosition()) return UntilLoop(scope, untilCondition, toPosition()) } -private fun Prog8ANTLRParser.WhenstmtContext.toAst(encoding: IStringEncoding): WhenStatement { - val condition = expression().toAst(encoding) - val choices = this.when_choice()?.map { it.toAst(encoding) }?.toMutableList() ?: mutableListOf() +private fun Prog8ANTLRParser.WhenstmtContext.toAst(): WhenStatement { + val condition = expression().toAst() + val choices = this.when_choice()?.map { it.toAst() }?.toMutableList() ?: mutableListOf() return WhenStatement(condition, choices, toPosition()) } -private fun Prog8ANTLRParser.When_choiceContext.toAst(encoding: IStringEncoding): WhenChoice { - val values = expression_list()?.toAst(encoding) - val stmt = statement()?.toAst(encoding) - val stmtBlock = statement_block()?.toAst(encoding)?.toMutableList() ?: mutableListOf() +private fun Prog8ANTLRParser.When_choiceContext.toAst(): WhenChoice { + val values = expression_list()?.toAst() + val stmt = statement()?.toAst() + val stmtBlock = statement_block()?.toAst()?.toMutableList() ?: mutableListOf() if(stmt!=null) stmtBlock.add(stmt) val scope = AnonymousScope(stmtBlock, toPosition()) return WhenChoice(values?.toMutableList(), scope, toPosition()) } -private fun Prog8ANTLRParser.VardeclContext.toAst(encoding: IStringEncoding): VarDecl { +private fun Prog8ANTLRParser.VardeclContext.toAst(): VarDecl { return VarDecl( VarDeclType.VAR, datatype()?.toAst() ?: DataType.UNDEFINED, if(ZEROPAGE() != null) ZeropageWish.PREFER_ZEROPAGE else ZeropageWish.DONTCARE, - arrayindex()?.toAst(encoding), + arrayindex()?.toAst(), varname.text, null, ARRAYSIG() != null || arrayindex() != null, diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index d54bc6925..78ac4f003 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -1,6 +1,5 @@ package prog8.parser -import prog8.ast.IStringEncoding import prog8.ast.Module import prog8.ast.Program import prog8.ast.base.Position @@ -17,7 +16,6 @@ fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') class ModuleImporter(private val program: Program, - private val encoder: IStringEncoding, private val compilationTargetName: String, private val libdirs: List) { diff --git a/compilerAst/src/prog8/parser/Petscii.kt b/compilerAst/src/prog8/parser/Petscii.kt deleted file mode 100644 index c79277af3..000000000 --- a/compilerAst/src/prog8/parser/Petscii.kt +++ /dev/null @@ -1,1171 +0,0 @@ -package prog8.parser - -import prog8.ast.antlr.escape -import java.io.CharConversionException - -object Petscii { - - // decoding: from Petscii/Screencodes (0-255) to unicode - // character tables used from https://github.com/dj51d/cbmcodecs - - private val decodingPetsciiLowercase = arrayOf( - '\u0000', // 0x00 -> \u0000 - '\ufffe', // 0x01 -> UNDEFINED - '\ufffe', // 0x02 -> UNDEFINED - '\ufffe', // 0x03 -> UNDEFINED - '\ufffe', // 0x04 -> UNDEFINED - '\uf100', // 0x05 -> WHITE COLOR SWITCH (CUS) - '\ufffe', // 0x06 -> UNDEFINED - '\ufffe', // 0x07 -> UNDEFINED - '\uf118', // 0x08 -> DISABLE CHARACTER SET SWITCHING (CUS) - '\uf119', // 0x09 -> ENABLE CHARACTER SET SWITCHING (CUS) - '\ufffe', // 0x0A -> UNDEFINED - '\ufffe', // 0x0B -> UNDEFINED - '\ufffe', // 0x0C -> UNDEFINED - '\r' , // 0x0D -> CARRIAGE RETURN - '\u000e', // 0x0E -> SHIFT OUT - '\ufffe', // 0x0F -> UNDEFINED - '\ufffe', // 0x10 -> UNDEFINED - '\uf11c', // 0x11 -> CURSOR DOWN (CUS) - '\uf11a', // 0x12 -> REVERSE VIDEO ON (CUS) - '\uf120', // 0x13 -> HOME (CUS) - '\u007f', // 0x14 -> DELETE - '\ufffe', // 0x15 -> UNDEFINED - '\ufffe', // 0x16 -> UNDEFINED - '\ufffe', // 0x17 -> UNDEFINED - '\ufffe', // 0x18 -> UNDEFINED - '\ufffe', // 0x19 -> UNDEFINED - '\ufffe', // 0x1A -> UNDEFINED - '\ufffe', // 0x1B -> UNDEFINED - '\uf101', // 0x1C -> RED COLOR SWITCH (CUS) - '\uf11d', // 0x1D -> CURSOR RIGHT (CUS) - '\uf102', // 0x1E -> GREEN COLOR SWITCH (CUS) - '\uf103', // 0x1F -> BLUE COLOR SWITCH (CUS) - ' ' , // 0x20 -> SPACE - '!' , // ! 0x21 -> EXCLAMATION MARK - '"' , // " 0x22 -> QUOTATION MARK - '#' , // # 0x23 -> NUMBER SIGN - '$' , // $ 0x24 -> DOLLAR SIGN - '%' , // % 0x25 -> PERCENT SIGN - '&' , // & 0x26 -> AMPERSAND - '\'' , // ' 0x27 -> APOSTROPHE - '(' , // ( 0x28 -> LEFT PARENTHESIS - ')' , // ) 0x29 -> RIGHT PARENTHESIS - '*' , // * 0x2A -> ASTERISK - '+' , // + 0x2B -> PLUS SIGN - ',' , // , 0x2C -> COMMA - '-' , // - 0x2D -> HYPHEN-MINUS - '.' , // . 0x2E -> FULL STOP - '/' , // / 0x2F -> SOLIDUS - '0' , // 0 0x30 -> DIGIT ZERO - '1' , // 1 0x31 -> DIGIT ONE - '2' , // 2 0x32 -> DIGIT TWO - '3' , // 3 0x33 -> DIGIT THREE - '4' , // 4 0x34 -> DIGIT FOUR - '5' , // 5 0x35 -> DIGIT FIVE - '6' , // 6 0x36 -> DIGIT SIX - '7' , // 7 0x37 -> DIGIT SEVEN - '8' , // 8 0x38 -> DIGIT EIGHT - '9' , // 9 0x39 -> DIGIT NINE - ':' , // : 0x3A -> COLON - ';' , // ; 0x3B -> SEMICOLON - '<' , // < 0x3C -> LESS-THAN SIGN - '=' , // = 0x3D -> EQUALS SIGN - '>' , // > 0x3E -> GREATER-THAN SIGN - '?' , // ? 0x3F -> QUESTION MARK - '@' , // @ 0x40 -> COMMERCIAL AT - 'a' , // a 0x41 -> LATIN SMALL LETTER A - 'b' , // b 0x42 -> LATIN SMALL LETTER B - 'c' , // c 0x43 -> LATIN SMALL LETTER C - 'd' , // d 0x44 -> LATIN SMALL LETTER D - 'e' , // e 0x45 -> LATIN SMALL LETTER E - 'f' , // f 0x46 -> LATIN SMALL LETTER F - 'g' , // g 0x47 -> LATIN SMALL LETTER G - 'h' , // h 0x48 -> LATIN SMALL LETTER H - 'i' , // i 0x49 -> LATIN SMALL LETTER I - 'j' , // j 0x4A -> LATIN SMALL LETTER J - 'k' , // k 0x4B -> LATIN SMALL LETTER K - 'l' , // l 0x4C -> LATIN SMALL LETTER L - 'm' , // m 0x4D -> LATIN SMALL LETTER M - 'n' , // n 0x4E -> LATIN SMALL LETTER N - 'o' , // o 0x4F -> LATIN SMALL LETTER O - 'p' , // p 0x50 -> LATIN SMALL LETTER P - 'q' , // q 0x51 -> LATIN SMALL LETTER Q - 'r' , // r 0x52 -> LATIN SMALL LETTER R - 's' , // s 0x53 -> LATIN SMALL LETTER S - 't' , // t 0x54 -> LATIN SMALL LETTER T - 'u' , // u 0x55 -> LATIN SMALL LETTER U - 'v' , // v 0x56 -> LATIN SMALL LETTER V - 'w' , // w 0x57 -> LATIN SMALL LETTER W - 'x' , // x 0x58 -> LATIN SMALL LETTER X - 'y' , // y 0x59 -> LATIN SMALL LETTER Y - 'z' , // z 0x5A -> LATIN SMALL LETTER Z - '[' , // [ 0x5B -> LEFT SQUARE BRACKET - '\u00a3', // £ 0x5C -> POUND SIGN - ']' , // ] 0x5D -> RIGHT SQUARE BRACKET - '\u2191', // ↑ 0x5E -> UPWARDS ARROW - '\u2190', // ← 0x5F -> LEFTWARDS ARROW - '\u2500', // ─ 0x60 -> BOX DRAWINGS LIGHT HORIZONTAL - 'A' , // A 0x61 -> LATIN CAPITAL LETTER A - 'B' , // B 0x62 -> LATIN CAPITAL LETTER B - 'C' , // C 0x63 -> LATIN CAPITAL LETTER C - 'D' , // D 0x64 -> LATIN CAPITAL LETTER D - 'E' , // E 0x65 -> LATIN CAPITAL LETTER E - 'F' , // F 0x66 -> LATIN CAPITAL LETTER F - 'G' , // G 0x67 -> LATIN CAPITAL LETTER G - 'H' , // H 0x68 -> LATIN CAPITAL LETTER H - 'I' , // I 0x69 -> LATIN CAPITAL LETTER I - 'J' , // J 0x6A -> LATIN CAPITAL LETTER J - 'K' , // K 0x6B -> LATIN CAPITAL LETTER K - 'L' , // L 0x6C -> LATIN CAPITAL LETTER L - 'M' , // M 0x6D -> LATIN CAPITAL LETTER M - 'N' , // N 0x6E -> LATIN CAPITAL LETTER N - 'O' , // O 0x6F -> LATIN CAPITAL LETTER O - 'P' , // P 0x70 -> LATIN CAPITAL LETTER P - 'Q' , // Q 0x71 -> LATIN CAPITAL LETTER Q - 'R' , // R 0x72 -> LATIN CAPITAL LETTER R - 'S' , // S 0x73 -> LATIN CAPITAL LETTER S - 'T' , // T 0x74 -> LATIN CAPITAL LETTER T - 'U' , // U 0x75 -> LATIN CAPITAL LETTER U - 'V' , // V 0x76 -> LATIN CAPITAL LETTER V - 'W' , // W 0x77 -> LATIN CAPITAL LETTER W - 'X' , // X 0x78 -> LATIN CAPITAL LETTER X - 'Y' , // Y 0x79 -> LATIN CAPITAL LETTER Y - 'Z' , // Z 0x7A -> LATIN CAPITAL LETTER Z - '\u253c', // ┼ 0x7B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0x7C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0x7D -> BOX DRAWINGS LIGHT VERTICAL - '\u2592', // ▒ 0x7E -> MEDIUM SHADE - '\uf139', //  0x7F -> MEDIUM SHADE SLASHED LEFT (CUS) - '\ufffe', // 0x80 -> UNDEFINED - '\uf104', // 0x81 -> ORANGE COLOR SWITCH (CUS) - '\ufffe', // 0x82 -> UNDEFINED - '\ufffe', // 0x83 -> UNDEFINED - '\ufffe', // 0x84 -> UNDEFINED - '\uf110', //  0x85 -> FUNCTION KEY 1 (CUS) - '\uf112', //  0x86 -> FUNCTION KEY 3 (CUS) - '\uf114', //  0x87 -> FUNCTION KEY 5 (CUS) - '\uf116', //  0x88 -> FUNCTION KEY 7 (CUS) - '\uf111', //  0x89 -> FUNCTION KEY 2 (CUS) - '\uf113', //  0x8A -> FUNCTION KEY 4 (CUS) - '\uf115', //  0x8B -> FUNCTION KEY 6 (CUS) - '\uf117', //  0x8C -> FUNCTION KEY 8 (CUS) - '\n' , // 0x8D -> LINE FEED - '\u000f', //  0x8E -> SHIFT IN - '\ufffe', // 0x8F -> UNDEFINED - '\uf105', // 0x90 -> BLACK COLOR SWITCH (CUS) - '\uf11e', //  0x91 -> CURSOR UP (CUS) - '\uf11b', //  0x92 -> REVERSE VIDEO OFF (CUS) - '\u000c', // 0x93 -> FORM FEED - '\uf121', //  0x94 -> INSERT (CUS) - '\uf106', // 0x95 -> BROWN COLOR SWITCH (CUS) - '\uf107', // 0x96 -> LIGHT RED COLOR SWITCH (CUS) - '\uf108', // 0x97 -> GRAY 1 COLOR SWITCH (CUS) - '\uf109', //  0x98 -> GRAY 2 COLOR SWITCH (CUS) - '\uf10a', //  0x99 -> LIGHT GREEN COLOR SWITCH (CUS) - '\uf10b', //  0x9A -> LIGHT BLUE COLOR SWITCH (CUS) - '\uf10c', //  0x9B -> GRAY 3 COLOR SWITCH (CUS) - '\uf10d', //  0x9C -> PURPLE COLOR SWITCH (CUS) - '\uf11d', //  0x9D -> CURSOR LEFT (CUS) - '\uf10e', //  0x9E -> YELLOW COLOR SWITCH (CUS) - '\uf10f', //  0x9F -> CYAN COLOR SWITCH (CUS) - '\u00a0', // 0xA0 -> NO-BREAK SPACE - '\u258c', // ▌ 0xA1 -> LEFT HALF BLOCK - '\u2584', // ▄ 0xA2 -> LOWER HALF BLOCK - '\u2594', // ▔ 0xA3 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0xA4 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0xA5 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0xA6 -> MEDIUM SHADE - '\u2595', // ▕ 0xA7 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0xA8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\uf13a', //  0xA9 -> MEDIUM SHADE SLASHED RIGHT (CUS) - '\uf130', //  0xAA -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0xAB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0xAC -> QUADRANT LOWER RIGHT - '\u2514', // └ 0xAD -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0xAE -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0xAF -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0xB0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0xB1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0xB2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0xB3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0xB4 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0xB5 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0xB6 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0xB7 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0xB8 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0xB9 -> LOWER THREE EIGHTHS BLOCK - '\u2713', // ✓ 0xBA -> CHECK MARK - '\u2596', // ▖ 0xBB -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0xBC -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0xBD -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0xBE -> QUADRANT UPPER LEFT - '\u259a', // ▚ 0xBF -> QUADRANT UPPER LEFT AND LOWER RIGHT - '\u2500', // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL - 'A' , // A 0xC1 -> LATIN CAPITAL LETTER A - 'B' , // B 0xC2 -> LATIN CAPITAL LETTER B - 'C' , // C 0xC3 -> LATIN CAPITAL LETTER C - 'D' , // D 0xC4 -> LATIN CAPITAL LETTER D - 'E' , // E 0xC5 -> LATIN CAPITAL LETTER E - 'F' , // F 0xC6 -> LATIN CAPITAL LETTER F - 'G' , // G 0xC7 -> LATIN CAPITAL LETTER G - 'H' , // H 0xC8 -> LATIN CAPITAL LETTER H - 'I' , // I 0xC9 -> LATIN CAPITAL LETTER I - 'J' , // J 0xCA -> LATIN CAPITAL LETTER J - 'K' , // K 0xCB -> LATIN CAPITAL LETTER K - 'L' , // L 0xCC -> LATIN CAPITAL LETTER L - 'M' , // M 0xCD -> LATIN CAPITAL LETTER M - 'N' , // N 0xCE -> LATIN CAPITAL LETTER N - 'O' , // O 0xCF -> LATIN CAPITAL LETTER O - 'P' , // P 0xD0 -> LATIN CAPITAL LETTER P - 'Q' , // Q 0xD1 -> LATIN CAPITAL LETTER Q - 'R' , // R 0xD2 -> LATIN CAPITAL LETTER R - 'S' , // S 0xD3 -> LATIN CAPITAL LETTER S - 'T' , // T 0xD4 -> LATIN CAPITAL LETTER T - 'U' , // U 0xD5 -> LATIN CAPITAL LETTER U - 'V' , // V 0xD6 -> LATIN CAPITAL LETTER V - 'W' , // W 0xD7 -> LATIN CAPITAL LETTER W - 'X' , // X 0xD8 -> LATIN CAPITAL LETTER X - 'Y' , // Y 0xD9 -> LATIN CAPITAL LETTER Y - 'Z' , // Z 0xDA -> LATIN CAPITAL LETTER Z - '\u253c', // ┼ 0xDB -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0xDC -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0xDD -> BOX DRAWINGS LIGHT VERTICAL - '\u2592', // ▒ 0xDE -> MEDIUM SHADE - '\uf139', //  0xDF -> MEDIUM SHADE SLASHED LEFT (CUS) - '\u00a0', // 0xE0 -> NO-BREAK SPACE - '\u258c', // ▌ 0xE1 -> LEFT HALF BLOCK - '\u2584', // ▄ 0xE2 -> LOWER HALF BLOCK - '\u2594', // ▔ 0xE3 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0xE4 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0xE5 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0xE6 -> MEDIUM SHADE - '\u2595', // ▕ 0xE7 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0xE8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\uf13a', //  0xE9 -> MEDIUM SHADE SLASHED RIGHT (CUS) - '\uf130', //  0xEA -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0xEB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0xEC -> QUADRANT LOWER RIGHT - '\u2514', // └ 0xED -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0xEE -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0xEF -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0xF0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0xF1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0xF2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0xF3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0xF4 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0xF5 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0xF6 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0xF7 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0xF8 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0xF9 -> LOWER THREE EIGHTHS BLOCK - '\u2713', // ✓ 0xFA -> CHECK MARK - '\u2596', // ▖ 0xFB -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0xFC -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0xFD -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0xFE -> QUADRANT UPPER LEFT - '\u2592' // ▒ 0xFF -> MEDIUM SHADE - ) - - private val decodingPetsciiUppercase = arrayOf( - '\u0000', // 0x00 -> \u0000 - '\ufffe', // 0x01 -> UNDEFINED - '\ufffe', // 0x02 -> UNDEFINED - '\ufffe', // 0x03 -> UNDEFINED - '\ufffe', // 0x04 -> UNDEFINED - '\uf100', // 0x05 -> WHITE COLOR SWITCH (CUS) - '\ufffe', // 0x06 -> UNDEFINED - '\ufffe', // 0x07 -> UNDEFINED - '\uf118', // 0x08 -> DISABLE CHARACTER SET SWITCHING (CUS) - '\uf119', // 0x09 -> ENABLE CHARACTER SET SWITCHING (CUS) - '\ufffe', // 0x0A -> UNDEFINED - '\ufffe', // 0x0B -> UNDEFINED - '\ufffe', // 0x0C -> UNDEFINED - '\r' , // 0x0D -> CARRIAGE RETURN - '\u000e', // 0x0E -> SHIFT OUT - '\ufffe', // 0x0F -> UNDEFINED - '\ufffe', // 0x10 -> UNDEFINED - '\uf11c', // 0x11 -> CURSOR DOWN (CUS) - '\uf11a', // 0x12 -> REVERSE VIDEO ON (CUS) - '\uf120', // 0x13 -> HOME (CUS) - '\u007f', // 0x14 -> DELETE - '\ufffe', // 0x15 -> UNDEFINED - '\ufffe', // 0x16 -> UNDEFINED - '\ufffe', // 0x17 -> UNDEFINED - '\ufffe', // 0x18 -> UNDEFINED - '\ufffe', // 0x19 -> UNDEFINED - '\ufffe', // 0x1A -> UNDEFINED - '\ufffe', // 0x1B -> UNDEFINED - '\uf101', // 0x1C -> RED COLOR SWITCH (CUS) - '\uf11d', // 0x1D -> CURSOR RIGHT (CUS) - '\uf102', // 0x1E -> GREEN COLOR SWITCH (CUS) - '\uf103', // 0x1F -> BLUE COLOR SWITCH (CUS) - ' ' , // 0x20 -> SPACE - '!' , // ! 0x21 -> EXCLAMATION MARK - '"' , // " 0x22 -> QUOTATION MARK - '#' , // # 0x23 -> NUMBER SIGN - '$' , // $ 0x24 -> DOLLAR SIGN - '%' , // % 0x25 -> PERCENT SIGN - '&' , // & 0x26 -> AMPERSAND - '\'' , // ' 0x27 -> APOSTROPHE - '(' , // ( 0x28 -> LEFT PARENTHESIS - ')' , // ) 0x29 -> RIGHT PARENTHESIS - '*' , // * 0x2A -> ASTERISK - '+' , // + 0x2B -> PLUS SIGN - ',' , // , 0x2C -> COMMA - '-' , // - 0x2D -> HYPHEN-MINUS - '.' , // . 0x2E -> FULL STOP - '/' , // / 0x2F -> SOLIDUS - '0' , // 0 0x30 -> DIGIT ZERO - '1' , // 1 0x31 -> DIGIT ONE - '2' , // 2 0x32 -> DIGIT TWO - '3' , // 3 0x33 -> DIGIT THREE - '4' , // 4 0x34 -> DIGIT FOUR - '5' , // 5 0x35 -> DIGIT FIVE - '6' , // 6 0x36 -> DIGIT SIX - '7' , // 7 0x37 -> DIGIT SEVEN - '8' , // 8 0x38 -> DIGIT EIGHT - '9' , // 9 0x39 -> DIGIT NINE - ':' , // : 0x3A -> COLON - ';' , // ; 0x3B -> SEMICOLON - '<' , // < 0x3C -> LESS-THAN SIGN - '=' , // = 0x3D -> EQUALS SIGN - '>' , // > 0x3E -> GREATER-THAN SIGN - '?' , // ? 0x3F -> QUESTION MARK - '@' , // @ 0x40 -> COMMERCIAL AT - 'A' , // A 0x41 -> LATIN CAPITAL LETTER A - 'B' , // B 0x42 -> LATIN CAPITAL LETTER B - 'C' , // C 0x43 -> LATIN CAPITAL LETTER C - 'D' , // D 0x44 -> LATIN CAPITAL LETTER D - 'E' , // E 0x45 -> LATIN CAPITAL LETTER E - 'F' , // F 0x46 -> LATIN CAPITAL LETTER F - 'G' , // G 0x47 -> LATIN CAPITAL LETTER G - 'H' , // H 0x48 -> LATIN CAPITAL LETTER H - 'I' , // I 0x49 -> LATIN CAPITAL LETTER I - 'J' , // J 0x4A -> LATIN CAPITAL LETTER J - 'K' , // K 0x4B -> LATIN CAPITAL LETTER K - 'L' , // L 0x4C -> LATIN CAPITAL LETTER L - 'M' , // M 0x4D -> LATIN CAPITAL LETTER M - 'N' , // N 0x4E -> LATIN CAPITAL LETTER N - 'O' , // O 0x4F -> LATIN CAPITAL LETTER O - 'P' , // P 0x50 -> LATIN CAPITAL LETTER P - 'Q' , // Q 0x51 -> LATIN CAPITAL LETTER Q - 'R' , // R 0x52 -> LATIN CAPITAL LETTER R - 'S' , // S 0x53 -> LATIN CAPITAL LETTER S - 'T' , // T 0x54 -> LATIN CAPITAL LETTER T - 'U' , // U 0x55 -> LATIN CAPITAL LETTER U - 'V' , // V 0x56 -> LATIN CAPITAL LETTER V - 'W' , // W 0x57 -> LATIN CAPITAL LETTER W - 'X' , // X 0x58 -> LATIN CAPITAL LETTER X - 'Y' , // Y 0x59 -> LATIN CAPITAL LETTER Y - 'Z' , // Z 0x5A -> LATIN CAPITAL LETTER Z - '[' , // [ 0x5B -> LEFT SQUARE BRACKET - '\u00a3', // £ 0x5C -> POUND SIGN - ']' , // ] 0x5D -> RIGHT SQUARE BRACKET - '\u2191', // ↑ 0x5E -> UPWARDS ARROW - '\u2190', // ← 0x5F -> LEFTWARDS ARROW - '\u2500', // ─ 0x60 -> BOX DRAWINGS LIGHT HORIZONTAL - '\u2660', // ♠ 0x61 -> BLACK SPADE SUIT - '\u2502', // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL - '\u2500', // ─ 0x63 -> BOX DRAWINGS LIGHT HORIZONTAL - '\uf122', //  0x64 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) - '\uf123', //  0x65 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) - '\uf124', //  0x66 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) - '\uf126', //  0x67 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) - '\uf128', //  0x68 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) - '\u256e', // ╮ 0x69 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT - '\u2570', // ╰ 0x6A -> BOX DRAWINGS LIGHT ARC UP AND RIGHT - '\u256f', // ╯ 0x6B -> BOX DRAWINGS LIGHT ARC UP AND LEFT - '\uf12a', //  0x6C -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) - '\u2572', // ╲ 0x6D -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT - '\u2571', // ╱ 0x6E -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT - '\uf12b', //  0x6F -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) - '\uf12c', //  0x70 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) - '\u25cf', // ● 0x71 -> BLACK CIRCLE - '\uf125', //  0x72 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) - '\u2665', // ♥ 0x73 -> BLACK HEART SUIT - '\uf127', //  0x74 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) - '\u256d', // ╭ 0x75 -> BOX DRAWINGS LIGHT ARC DOWN AND RIGHT - '\u2573', // ╳ 0x76 -> BOX DRAWINGS LIGHT DIAGONAL CROSS - '\u25cb', // ○ 0x77 -> WHITE CIRCLE - '\u2663', // ♣ 0x78 -> BLACK CLUB SUIT - '\uf129', //  0x79 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) - '\u2666', // ♦ 0x7A -> BLACK DIAMOND SUIT - '\u253c', // ┼ 0x7B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0x7C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0x7D -> BOX DRAWINGS LIGHT VERTICAL - '\u03c0', // π 0x7E -> GREEK SMALL LETTER PI - '\u25e5', // ◥ 0x7F -> BLACK UPPER RIGHT TRIANGLE - '\ufffe', // 0x80 -> UNDEFINED - '\uf104', //  0x81 -> ORANGE COLOR SWITCH (CUS) - '\ufffe', // 0x82 -> UNDEFINED - '\ufffe', // 0x83 -> UNDEFINED - '\ufffe', // 0x84 -> UNDEFINED - '\uf110', // 0x85 -> FUNCTION KEY 1 (CUS) - '\uf112', // 0x86 -> FUNCTION KEY 3 (CUS) - '\uf114', // 0x87 -> FUNCTION KEY 5 (CUS) - '\uf116', // 0x88 -> FUNCTION KEY 7 (CUS) - '\uf111', // 0x89 -> FUNCTION KEY 2 (CUS) - '\uf113', // 0x8A -> FUNCTION KEY 4 (CUS) - '\uf115', // 0x8B -> FUNCTION KEY 6 (CUS) - '\uf117', // 0x8C -> FUNCTION KEY 8 (CUS) - '\n' , // 0x8D -> LINE FEED - '\u000f', // 0x8E -> SHIFT IN - '\ufffe', // 0x8F -> UNDEFINED - '\uf105', // 0x90 -> BLACK COLOR SWITCH (CUS) - '\uf11e', // 0x91 -> CURSOR UP (CUS) - '\uf11b', // 0x92 -> REVERSE VIDEO OFF (CUS) - '\u000c', // 0x93 -> FORM FEED - '\uf121', // 0x94 -> INSERT (CUS) - '\uf106', // 0x95 -> BROWN COLOR SWITCH (CUS) - '\uf107', // 0x96 -> LIGHT RED COLOR SWITCH (CUS) - '\uf108', // 0x97 -> GRAY 1 COLOR SWITCH (CUS) - '\uf109', // 0x98 -> GRAY 2 COLOR SWITCH (CUS) - '\uf10a', // 0x99 -> LIGHT GREEN COLOR SWITCH (CUS) - '\uf10b', // 0x9A -> LIGHT BLUE COLOR SWITCH (CUS) - '\uf10c', // 0x9B -> GRAY 3 COLOR SWITCH (CUS) - '\uf10d', // 0x9C -> PURPLE COLOR SWITCH (CUS) - '\uf11d', // 0x9D -> CURSOR LEFT (CUS) - '\uf10e', // 0x9E -> YELLOW COLOR SWITCH (CUS) - '\uf10f', // 0x9F -> CYAN COLOR SWITCH (CUS) - '\u00a0', // 0xA0 -> NO-BREAK SPACE - '\u258c', // ▌ 0xA1 -> LEFT HALF BLOCK - '\u2584', // ▄ 0xA2 -> LOWER HALF BLOCK - '\u2594', // ▔ 0xA3 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0xA4 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0xA5 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0xA6 -> MEDIUM SHADE - '\u2595', // ▕ 0xA7 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0xA8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\u25e4', // ◤ 0xA9 -> BLACK UPPER LEFT TRIANGLE - '\uf130', //  0xAA -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0xAB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0xAC -> QUADRANT LOWER RIGHT - '\u2514', // └ 0xAD -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0xAE -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0xAF -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0xB0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0xB1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0xB2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0xB3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0xB4 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0xB5 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0xB6 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0xB7 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0xB8 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0xB9 -> LOWER THREE EIGHTHS BLOCK - '\uf12d', //  0xBA -> ONE EIGHTH BLOCK UP AND LEFT (CUS) - '\u2596', // ▖ 0xBB -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0xBC -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0xBD -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0xBE -> QUADRANT UPPER LEFT - '\u259a', // ▚ 0xBF -> QUADRANT UPPER LEFT AND LOWER RIGHT - '\u2500', // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL - '\u2660', // ♠ 0xC1 -> BLACK SPADE SUIT - '\u2502', // │ 0xC2 -> BOX DRAWINGS LIGHT VERTICAL - '\u2500', // ─ 0xC3 -> BOX DRAWINGS LIGHT HORIZONTAL - '\uf122', //  0xC4 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) - '\uf123', //  0xC5 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) - '\uf124', //  0xC6 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) - '\uf126', //  0xC7 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) - '\uf128', //  0xC8 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) - '\u256e', // ╮ 0xC9 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT - '\u2570', // ╰ 0xCA -> BOX DRAWINGS LIGHT ARC UP AND RIGHT - '\u256f', // ╯ 0xCB -> BOX DRAWINGS LIGHT ARC UP AND LEFT - '\uf12a', //  0xCC -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) - '\u2572', // ╲ 0xCD -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT - '\u2571', // ╱ 0xCE -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT - '\uf12b', //  0xCF -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) - '\uf12c', //  0xD0 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) - '\u25cf', // ● 0xD1 -> BLACK CIRCLE - '\uf125', //  0xD2 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) - '\u2665', // ♥ 0xD3 -> BLACK HEART SUIT - '\uf127', //  0xD4 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) - '\u256d', // ╭ 0xD5 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT - '\u2573', // ╳ 0xD6 -> BOX DRAWINGS LIGHT DIAGONAL CROSS - '\u25cb', // ○ 0xD7 -> WHITE CIRCLE - '\u2663', // ♣ 0xD8 -> BLACK CLUB SUIT - '\uf129', //  0xD9 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) - '\u2666', // ♦ 0xDA -> BLACK DIAMOND SUIT - '\u253c', // ┼ 0xDB -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0xDC -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0xDD -> BOX DRAWINGS LIGHT VERTICAL - '\u03c0', // π 0xDE -> GREEK SMALL LETTER PI - '\u25e5', // ◥ 0xDF -> BLACK UPPER RIGHT TRIANGLE - '\u00a0', // 0xE0 -> NO-BREAK SPACE - '\u258c', // ▌ 0xE1 -> LEFT HALF BLOCK - '\u2584', // ▄ 0xE2 -> LOWER HALF BLOCK - '\u2594', // ▔ 0xE3 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0xE4 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0xE5 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0xE6 -> MEDIUM SHADE - '\u2595', // ▕ 0xE7 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0xE8 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\u25e4', // ◤ 0xE9 -> BLACK UPPER LEFT TRIANGLE - '\uf130', //  0xEA -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0xEB -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0xEC -> QUADRANT LOWER RIGHT - '\u2514', // └ 0xED -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0xEE -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0xEF -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0xF0 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0xF1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0xF2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0xF3 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0xF4 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0xF5 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0xF6 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0xF7 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0xF8 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0xF9 -> LOWER THREE EIGHTHS BLOCK - '\uf12d', //  0xFA -> ONE EIGHTH BLOCK UP AND LEFT (CUS) - '\u2596', // ▖ 0xFB -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0xFC -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0xFD -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0xFE -> QUADRANT UPPER LEFT - '\u03c0' // π 0xFF -> GREEK SMALL LETTER PI - ) - - private val decodingScreencodeLowercase = arrayOf( - '@' , // @ 0x00 -> COMMERCIAL AT - 'a' , // a 0x01 -> LATIN SMALL LETTER A - 'b' , // b 0x02 -> LATIN SMALL LETTER B - 'c' , // c 0x03 -> LATIN SMALL LETTER C - 'd' , // d 0x04 -> LATIN SMALL LETTER D - 'e' , // e 0x05 -> LATIN SMALL LETTER E - 'f' , // f 0x06 -> LATIN SMALL LETTER F - 'g' , // g 0x07 -> LATIN SMALL LETTER G - 'h' , // h 0x08 -> LATIN SMALL LETTER H - 'i' , // i 0x09 -> LATIN SMALL LETTER I - 'j' , // j 0x0A -> LATIN SMALL LETTER J - 'k' , // k 0x0B -> LATIN SMALL LETTER K - 'l' , // l 0x0C -> LATIN SMALL LETTER L - 'm' , // m 0x0D -> LATIN SMALL LETTER M - 'n' , // n 0x0E -> LATIN SMALL LETTER N - 'o' , // o 0x0F -> LATIN SMALL LETTER O - 'p' , // p 0x10 -> LATIN SMALL LETTER P - 'q' , // q 0x11 -> LATIN SMALL LETTER Q - 'r' , // r 0x12 -> LATIN SMALL LETTER R - 's' , // s 0x13 -> LATIN SMALL LETTER S - 't' , // t 0x14 -> LATIN SMALL LETTER T - 'u' , // u 0x15 -> LATIN SMALL LETTER U - 'v' , // v 0x16 -> LATIN SMALL LETTER V - 'w' , // w 0x17 -> LATIN SMALL LETTER W - 'x' , // x 0x18 -> LATIN SMALL LETTER X - 'y' , // y 0x19 -> LATIN SMALL LETTER Y - 'z' , // z 0x1A -> LATIN SMALL LETTER Z - '[' , // [ 0x1B -> LEFT SQUARE BRACKET - '\u00a3', // £ 0x1C -> POUND SIGN - ']' , // ] 0x1D -> RIGHT SQUARE BRACKET - '\u2191', // ↑ 0x1E -> UPWARDS ARROW - '\u2190', // ← 0x1F -> LEFTWARDS ARROW - ' ' , // 0x20 -> SPACE - '!' , // ! 0x21 -> EXCLAMATION MARK - '"' , // " 0x22 -> QUOTATION MARK - '#' , // # 0x23 -> NUMBER SIGN - '$' , // $ 0x24 -> DOLLAR SIGN - '%' , // % 0x25 -> PERCENT SIGN - '&' , // & 0x26 -> AMPERSAND - '\'' , // ' 0x27 -> APOSTROPHE - '(' , // ( 0x28 -> LEFT PARENTHESIS - ')' , // ) 0x29 -> RIGHT PARENTHESIS - '*' , // * 0x2A -> ASTERISK - '+' , // + 0x2B -> PLUS SIGN - ',' , // , 0x2C -> COMMA - '-' , // - 0x2D -> HYPHEN-MINUS - '.' , // . 0x2E -> FULL STOP - '/' , // / 0x2F -> SOLIDUS - '0' , // 0 0x30 -> DIGIT ZERO - '1' , // 1 0x31 -> DIGIT ONE - '2' , // 2 0x32 -> DIGIT TWO - '3' , // 3 0x33 -> DIGIT THREE - '4' , // 4 0x34 -> DIGIT FOUR - '5' , // 5 0x35 -> DIGIT FIVE - '6' , // 6 0x36 -> DIGIT SIX - '7' , // 7 0x37 -> DIGIT SEVEN - '8' , // 8 0x38 -> DIGIT EIGHT - '9' , // 9 0x39 -> DIGIT NINE - ':' , // : 0x3A -> COLON - ';' , // ; 0x3B -> SEMICOLON - '<' , // < 0x3C -> LESS-THAN SIGN - '=' , // = 0x3D -> EQUALS SIGN - '>' , // > 0x3E -> GREATER-THAN SIGN - '?' , // ? 0x3F -> QUESTION MARK - '\u2500', // ─ 0x40 -> BOX DRAWINGS LIGHT HORIZONTAL - 'A' , // A 0x41 -> LATIN CAPITAL LETTER A - 'B' , // B 0x42 -> LATIN CAPITAL LETTER B - 'C' , // C 0x43 -> LATIN CAPITAL LETTER C - 'D' , // D 0x44 -> LATIN CAPITAL LETTER D - 'E' , // E 0x45 -> LATIN CAPITAL LETTER E - 'F' , // F 0x46 -> LATIN CAPITAL LETTER F - 'G' , // G 0x47 -> LATIN CAPITAL LETTER G - 'H' , // H 0x48 -> LATIN CAPITAL LETTER H - 'I' , // I 0x49 -> LATIN CAPITAL LETTER I - 'J' , // J 0x4A -> LATIN CAPITAL LETTER J - 'K' , // K 0x4B -> LATIN CAPITAL LETTER K - 'L' , // L 0x4C -> LATIN CAPITAL LETTER L - 'M' , // M 0x4D -> LATIN CAPITAL LETTER M - 'N' , // N 0x4E -> LATIN CAPITAL LETTER N - 'O' , // O 0x4F -> LATIN CAPITAL LETTER O - 'P' , // P 0x50 -> LATIN CAPITAL LETTER P - 'Q' , // Q 0x51 -> LATIN CAPITAL LETTER Q - 'R' , // R 0x52 -> LATIN CAPITAL LETTER R - 'S' , // S 0x53 -> LATIN CAPITAL LETTER S - 'T' , // T 0x54 -> LATIN CAPITAL LETTER T - 'U' , // U 0x55 -> LATIN CAPITAL LETTER U - 'V' , // V 0x56 -> LATIN CAPITAL LETTER V - 'W' , // W 0x57 -> LATIN CAPITAL LETTER W - 'X' , // X 0x58 -> LATIN CAPITAL LETTER X - 'Y' , // Y 0x59 -> LATIN CAPITAL LETTER Y - 'Z' , // Z 0x5A -> LATIN CAPITAL LETTER Z - '\u253c', // ┼ 0x5B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0x5C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0x5D -> BOX DRAWINGS LIGHT VERTICAL - '\u2592', // ▒ 0x5E -> MEDIUM SHADE - '\uf139', //  0x5F -> MEDIUM SHADE SLASHED LEFT (CUS) - '\u00a0', // 0x60 -> NO-BREAK SPACE - '\u258c', // ▌ 0x61 -> LEFT HALF BLOCK - '\u2584', // ▄ 0x62 -> LOWER HALF BLOCK - '\u2594', // ▔ 0x63 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0x64 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0x65 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0x66 -> MEDIUM SHADE - '\u2595', // ▕ 0x67 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0x68 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\uf13a', //  0x69 -> MEDIUM SHADE SLASHED RIGHT (CUS) - '\uf130', //  0x6A -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0x6B -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0x6C -> QUADRANT LOWER RIGHT - '\u2514', // └ 0x6D -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0x6E -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0x6F -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0x70 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0x71 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0x72 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0x73 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0x74 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0x75 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0x76 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0x77 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0x78 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0x79 -> LOWER THREE EIGHTHS BLOCK - '\u2713', // ✓ 0x7A -> CHECK MARK - '\u2596', // ▖ 0x7B -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0x7C -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0x7D -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0x7E -> QUADRANT UPPER LEFT - '\u259a', // ▚ 0x7F -> QUADRANT UPPER LEFT AND LOWER RIGHT - '\ufffe', // 0x80 -> UNDEFINED - '\ufffe', // 0x81 -> UNDEFINED - '\ufffe', // 0x82 -> UNDEFINED - '\ufffe', // 0x83 -> UNDEFINED - '\ufffe', // 0x84 -> UNDEFINED - '\ufffe', // 0x85 -> UNDEFINED - '\ufffe', // 0x86 -> UNDEFINED - '\ufffe', // 0x87 -> UNDEFINED - '\ufffe', // 0x88 -> UNDEFINED - '\ufffe', // 0x89 -> UNDEFINED - '\ufffe', // 0x8A -> UNDEFINED - '\ufffe', // 0x8B -> UNDEFINED - '\ufffe', // 0x8C -> UNDEFINED - '\ufffe', // 0x8D -> UNDEFINED - '\ufffe', // 0x8E -> UNDEFINED - '\ufffe', // 0x8F -> UNDEFINED - '\ufffe', // 0x90 -> UNDEFINED - '\ufffe', // 0x91 -> UNDEFINED - '\ufffe', // 0x92 -> UNDEFINED - '\ufffe', // 0x93 -> UNDEFINED - '\ufffe', // 0x94 -> UNDEFINED - '\ufffe', // 0x95 -> UNDEFINED - '\ufffe', // 0x96 -> UNDEFINED - '\ufffe', // 0x97 -> UNDEFINED - '\ufffe', // 0x98 -> UNDEFINED - '\ufffe', // 0x99 -> UNDEFINED - '\ufffe', // 0x9A -> UNDEFINED - '\ufffe', // 0x9B -> UNDEFINED - '\ufffe', // 0x9C -> UNDEFINED - '\ufffe', // 0x9D -> UNDEFINED - '\ufffe', // 0x9E -> UNDEFINED - '\ufffe', // 0x9F -> UNDEFINED - '\ufffe', // 0xA0 -> UNDEFINED - '\ufffe', // 0xA1 -> UNDEFINED - '\ufffe', // 0xA2 -> UNDEFINED - '\ufffe', // 0xA3 -> UNDEFINED - '\ufffe', // 0xA4 -> UNDEFINED - '\ufffe', // 0xA5 -> UNDEFINED - '\ufffe', // 0xA6 -> UNDEFINED - '\ufffe', // 0xA7 -> UNDEFINED - '\ufffe', // 0xA8 -> UNDEFINED - '\ufffe', // 0xA9 -> UNDEFINED - '\ufffe', // 0xAA -> UNDEFINED - '\ufffe', // 0xAB -> UNDEFINED - '\ufffe', // 0xAC -> UNDEFINED - '\ufffe', // 0xAD -> UNDEFINED - '\ufffe', // 0xAE -> UNDEFINED - '\ufffe', // 0xAF -> UNDEFINED - '\ufffe', // 0xB0 -> UNDEFINED - '\ufffe', // 0xB1 -> UNDEFINED - '\ufffe', // 0xB2 -> UNDEFINED - '\ufffe', // 0xB3 -> UNDEFINED - '\ufffe', // 0xB4 -> UNDEFINED - '\ufffe', // 0xB5 -> UNDEFINED - '\ufffe', // 0xB6 -> UNDEFINED - '\ufffe', // 0xB7 -> UNDEFINED - '\ufffe', // 0xB8 -> UNDEFINED - '\ufffe', // 0xB9 -> UNDEFINED - '\ufffe', // 0xBA -> UNDEFINED - '\ufffe', // 0xBB -> UNDEFINED - '\ufffe', // 0xBC -> UNDEFINED - '\ufffe', // 0xBD -> UNDEFINED - '\ufffe', // 0xBE -> UNDEFINED - '\ufffe', // 0xBF -> UNDEFINED - '\ufffe', // 0xC0 -> UNDEFINED - '\ufffe', // 0xC1 -> UNDEFINED - '\ufffe', // 0xC2 -> UNDEFINED - '\ufffe', // 0xC3 -> UNDEFINED - '\ufffe', // 0xC4 -> UNDEFINED - '\ufffe', // 0xC5 -> UNDEFINED - '\ufffe', // 0xC6 -> UNDEFINED - '\ufffe', // 0xC7 -> UNDEFINED - '\ufffe', // 0xC8 -> UNDEFINED - '\ufffe', // 0xC9 -> UNDEFINED - '\ufffe', // 0xCA -> UNDEFINED - '\ufffe', // 0xCB -> UNDEFINED - '\ufffe', // 0xCC -> UNDEFINED - '\ufffe', // 0xCD -> UNDEFINED - '\ufffe', // 0xCE -> UNDEFINED - '\ufffe', // 0xCF -> UNDEFINED - '\ufffe', // 0xD0 -> UNDEFINED - '\ufffe', // 0xD1 -> UNDEFINED - '\ufffe', // 0xD2 -> UNDEFINED - '\ufffe', // 0xD3 -> UNDEFINED - '\ufffe', // 0xD4 -> UNDEFINED - '\ufffe', // 0xD5 -> UNDEFINED - '\ufffe', // 0xD6 -> UNDEFINED - '\ufffe', // 0xD7 -> UNDEFINED - '\ufffe', // 0xD8 -> UNDEFINED - '\ufffe', // 0xD9 -> UNDEFINED - '\ufffe', // 0xDA -> UNDEFINED - '\ufffe', // 0xDB -> UNDEFINED - '\ufffe', // 0xDC -> UNDEFINED - '\ufffe', // 0xDD -> UNDEFINED - '\ufffe', // 0xDE -> UNDEFINED - '\ufffe', // 0xDF -> UNDEFINED - '\ufffe', // 0xE0 -> UNDEFINED - '\ufffe', // 0xE1 -> UNDEFINED - '\ufffe', // 0xE2 -> UNDEFINED - '\ufffe', // 0xE3 -> UNDEFINED - '\ufffe', // 0xE4 -> UNDEFINED - '\ufffe', // 0xE5 -> UNDEFINED - '\ufffe', // 0xE6 -> UNDEFINED - '\ufffe', // 0xE7 -> UNDEFINED - '\ufffe', // 0xE8 -> UNDEFINED - '\ufffe', // 0xE9 -> UNDEFINED - '\ufffe', // 0xEA -> UNDEFINED - '\ufffe', // 0xEB -> UNDEFINED - '\ufffe', // 0xEC -> UNDEFINED - '\ufffe', // 0xED -> UNDEFINED - '\ufffe', // 0xEE -> UNDEFINED - '\ufffe', // 0xEF -> UNDEFINED - '\ufffe', // 0xF0 -> UNDEFINED - '\ufffe', // 0xF1 -> UNDEFINED - '\ufffe', // 0xF2 -> UNDEFINED - '\ufffe', // 0xF3 -> UNDEFINED - '\ufffe', // 0xF4 -> UNDEFINED - '\ufffe', // 0xF5 -> UNDEFINED - '\ufffe', // 0xF6 -> UNDEFINED - '\ufffe', // 0xF7 -> UNDEFINED - '\ufffe', // 0xF8 -> UNDEFINED - '\ufffe', // 0xF9 -> UNDEFINED - '\ufffe', // 0xFA -> UNDEFINED - '\ufffe', // 0xFB -> UNDEFINED - '\ufffe', // 0xFC -> UNDEFINED - '\ufffe', // 0xFD -> UNDEFINED - '\ufffe', // 0xFE -> UNDEFINED - '\ufffe' // 0xFF -> UNDEFINED - ) - - private val decodingScreencodeUppercase = arrayOf( - '@' , // @ 0x00 -> COMMERCIAL AT - 'A' , // A 0x01 -> LATIN CAPITAL LETTER A - 'B' , // B 0x02 -> LATIN CAPITAL LETTER B - 'C' , // C 0x03 -> LATIN CAPITAL LETTER C - 'D' , // D 0x04 -> LATIN CAPITAL LETTER D - 'E' , // E 0x05 -> LATIN CAPITAL LETTER E - 'F' , // F 0x06 -> LATIN CAPITAL LETTER F - 'G' , // G 0x07 -> LATIN CAPITAL LETTER G - 'H' , // H 0x08 -> LATIN CAPITAL LETTER H - 'I' , // I 0x09 -> LATIN CAPITAL LETTER I - 'J' , // J 0x0A -> LATIN CAPITAL LETTER J - 'K' , // K 0x0B -> LATIN CAPITAL LETTER K - 'L' , // L 0x0C -> LATIN CAPITAL LETTER L - 'M' , // M 0x0D -> LATIN CAPITAL LETTER M - 'N' , // N 0x0E -> LATIN CAPITAL LETTER N - 'O' , // O 0x0F -> LATIN CAPITAL LETTER O - 'P' , // P 0x10 -> LATIN CAPITAL LETTER P - 'Q' , // Q 0x11 -> LATIN CAPITAL LETTER Q - 'R' , // R 0x12 -> LATIN CAPITAL LETTER R - 'S' , // S 0x13 -> LATIN CAPITAL LETTER S - 'T' , // T 0x14 -> LATIN CAPITAL LETTER T - 'U' , // U 0x15 -> LATIN CAPITAL LETTER U - 'V' , // V 0x16 -> LATIN CAPITAL LETTER V - 'W' , // W 0x17 -> LATIN CAPITAL LETTER W - 'X' , // X 0x18 -> LATIN CAPITAL LETTER X - 'Y' , // Y 0x19 -> LATIN CAPITAL LETTER Y - 'Z' , // Z 0x1A -> LATIN CAPITAL LETTER Z - '[' , // [ 0x1B -> LEFT SQUARE BRACKET - '\u00a3', // £ 0x1C -> POUND SIGN - ']' , // ] 0x1D -> RIGHT SQUARE BRACKET - '\u2191', // ↑ 0x1E -> UPWARDS ARROW - '\u2190', // ← 0x1F -> LEFTWARDS ARROW - ' ' , // 0x20 -> SPACE - '!' , // ! 0x21 -> EXCLAMATION MARK - '"' , // " 0x22 -> QUOTATION MARK - '#' , // # 0x23 -> NUMBER SIGN - '$' , // $ 0x24 -> DOLLAR SIGN - '%' , // % 0x25 -> PERCENT SIGN - '&' , // & 0x26 -> AMPERSAND - '\'' , // ' 0x27 -> APOSTROPHE - '(' , // ( 0x28 -> LEFT PARENTHESIS - ')' , // ) 0x29 -> RIGHT PARENTHESIS - '*' , // * 0x2A -> ASTERISK - '+' , // + 0x2B -> PLUS SIGN - ',' , // , 0x2C -> COMMA - '-' , // - 0x2D -> HYPHEN-MINUS - '.' , // . 0x2E -> FULL STOP - '/' , // / 0x2F -> SOLIDUS - '0' , // 0 0x30 -> DIGIT ZERO - '1' , // 1 0x31 -> DIGIT ONE - '2' , // 2 0x32 -> DIGIT TWO - '3' , // 3 0x33 -> DIGIT THREE - '4' , // 4 0x34 -> DIGIT FOUR - '5' , // 5 0x35 -> DIGIT FIVE - '6' , // 6 0x36 -> DIGIT SIX - '7' , // 7 0x37 -> DIGIT SEVEN - '8' , // 8 0x38 -> DIGIT EIGHT - '9' , // 9 0x39 -> DIGIT NINE - ':' , // : 0x3A -> COLON - ';' , // ; 0x3B -> SEMICOLON - '<' , // < 0x3C -> LESS-THAN SIGN - '=' , // = 0x3D -> EQUALS SIGN - '>' , // > 0x3E -> GREATER-THAN SIGN - '?' , // ? 0x3F -> QUESTION MARK - '\u2500', // ─ 0x40 -> BOX DRAWINGS LIGHT HORIZONTAL - '\u2660', // ♠ 0x41 -> BLACK SPADE SUIT - '\u2502', // │ 0x42 -> BOX DRAWINGS LIGHT VERTICAL - '\u2500', // ─ 0x43 -> BOX DRAWINGS LIGHT HORIZONTAL - '\uf122', //  0x44 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER UP (CUS) - '\uf123', //  0x45 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS UP (CUS) - '\uf124', //  0x46 -> BOX DRAWINGS LIGHT HORIZONTAL ONE QUARTER DOWN (CUS) - '\uf126', //  0x47 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER LEFT (CUS) - '\uf128', //  0x48 -> BOX DRAWINGS LIGHT VERTICAL ONE QUARTER RIGHT (CUS) - '\u256e', // ╮ 0x49 -> BOX DRAWINGS LIGHT ARC DOWN AND LEFT - '\u2570', // ╰ 0x4A -> BOX DRAWINGS LIGHT ARC UP AND RIGHT - '\u256f', // ╯ 0x4B -> BOX DRAWINGS LIGHT ARC UP AND LEFT - '\uf12a', //  0x4C -> ONE EIGHTH BLOCK UP AND RIGHT (CUS) - '\u2572', // ╲ 0x4D -> BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT - '\u2571', // ╱ 0x4E -> BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT - '\uf12b', //  0x4F -> ONE EIGHTH BLOCK DOWN AND RIGHT (CUS) - '\uf12c', //  0x50 -> ONE EIGHTH BLOCK DOWN AND LEFT (CUS) - '\u25cf', // ● 0x51 -> BLACK CIRCLE - '\uf125', //  0x52 -> BOX DRAWINGS LIGHT HORIZONTAL TWO QUARTERS DOWN (CUS) - '\u2665', // ♥ 0x53 -> BLACK HEART SUIT - '\uf127', //  0x54 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS LEFT (CUS) - '\u256d', // ╭ 0x55 -> BOX DRAWINGS LIGHT ARC DOWN AND RIGHT - '\u2573', // ╳ 0x56 -> BOX DRAWINGS LIGHT DIAGONAL CROSS - '\u25cb', // ○ 0x57 -> WHITE CIRCLE - '\u2663', // ♣ 0x58 -> BLACK CLUB SUIT - '\uf129', //  0x59 -> BOX DRAWINGS LIGHT VERTICAL TWO QUARTERS RIGHT (CUS) - '\u2666', // ♦ 0x5A -> BLACK DIAMOND SUIT - '\u253c', // ┼ 0x5B -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\uf12e', //  0x5C -> LEFT HALF BLOCK MEDIUM SHADE (CUS) - '\u2502', // │ 0x5D -> BOX DRAWINGS LIGHT VERTICAL - '\u03c0', // π 0x5E -> GREEK SMALL LETTER PI - '\u25e5', // ◥ 0x5F -> BLACK UPPER RIGHT TRIANGLE - '\u00a0', // 0x60 -> NO-BREAK SPACE - '\u258c', // ▌ 0x61 -> LEFT HALF BLOCK - '\u2584', // ▄ 0x62 -> LOWER HALF BLOCK - '\u2594', // ▔ 0x63 -> UPPER ONE EIGHTH BLOCK - '\u2581', // ▁ 0x64 -> LOWER ONE EIGHTH BLOCK - '\u258f', // ▏ 0x65 -> LEFT ONE EIGHTH BLOCK - '\u2592', // ▒ 0x66 -> MEDIUM SHADE - '\u2595', // ▕ 0x67 -> RIGHT ONE EIGHTH BLOCK - '\uf12f', //  0x68 -> LOWER HALF BLOCK MEDIUM SHADE (CUS) - '\u25e4', // ◤ 0x69 -> BLACK UPPER LEFT TRIANGLE - '\uf130', //  0x6A -> RIGHT ONE QUARTER BLOCK (CUS) - '\u251c', // ├ 0x6B -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2597', // ▗ 0x6C -> QUADRANT LOWER RIGHT - '\u2514', // └ 0x6D -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2510', // ┐ 0x6E -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2582', // ▂ 0x6F -> LOWER ONE QUARTER BLOCK - '\u250c', // ┌ 0x70 -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2534', // ┴ 0x71 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c', // ┬ 0x72 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u2524', // ┤ 0x73 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u258e', // ▎ 0x74 -> LEFT ONE QUARTER BLOCK - '\u258d', // ▍ 0x75 -> LEFT THREE EIGTHS BLOCK - '\uf131', //  0x76 -> RIGHT THREE EIGHTHS BLOCK (CUS) - '\uf132', //  0x77 -> UPPER ONE QUARTER BLOCK (CUS) - '\uf133', //  0x78 -> UPPER THREE EIGHTS BLOCK (CUS) - '\u2583', // ▃ 0x79 -> LOWER THREE EIGHTHS BLOCK - '\uf12d', //  0x7A -> ONE EIGHTH BLOCK UP AND LEFT (CUS) - '\u2596', // ▖ 0x7B -> QUADRANT LOWER LEFT - '\u259d', // ▝ 0x7C -> QUADRANT UPPER RIGHT - '\u2518', // ┘ 0x7D -> BOX DRAWINGS LIGHT UP AND LEFT - '\u2598', // ▘ 0x7E -> QUADRANT UPPER LEFT - '\u259a', // ▚ 0x7F -> QUADRANT UPPER LEFT AND LOWER RIGHT - '\ufffe', // 0x80 -> UNDEFINED - '\ufffe', // 0x81 -> UNDEFINED - '\ufffe', // 0x82 -> UNDEFINED - '\ufffe', // 0x83 -> UNDEFINED - '\ufffe', // 0x84 -> UNDEFINED - '\ufffe', // 0x85 -> UNDEFINED - '\ufffe', // 0x86 -> UNDEFINED - '\ufffe', // 0x87 -> UNDEFINED - '\ufffe', // 0x88 -> UNDEFINED - '\ufffe', // 0x89 -> UNDEFINED - '\ufffe', // 0x8A -> UNDEFINED - '\ufffe', // 0x8B -> UNDEFINED - '\ufffe', // 0x8C -> UNDEFINED - '\ufffe', // 0x8D -> UNDEFINED - '\ufffe', // 0x8E -> UNDEFINED - '\ufffe', // 0x8F -> UNDEFINED - '\ufffe', // 0x90 -> UNDEFINED - '\ufffe', // 0x91 -> UNDEFINED - '\ufffe', // 0x92 -> UNDEFINED - '\ufffe', // 0x93 -> UNDEFINED - '\ufffe', // 0x94 -> UNDEFINED - '\ufffe', // 0x95 -> UNDEFINED - '\ufffe', // 0x96 -> UNDEFINED - '\ufffe', // 0x97 -> UNDEFINED - '\ufffe', // 0x98 -> UNDEFINED - '\ufffe', // 0x99 -> UNDEFINED - '\ufffe', // 0x9A -> UNDEFINED - '\ufffe', // 0x9B -> UNDEFINED - '\ufffe', // 0x9C -> UNDEFINED - '\ufffe', // 0x9D -> UNDEFINED - '\ufffe', // 0x9E -> UNDEFINED - '\ufffe', // 0x9F -> UNDEFINED - '\ufffe', // 0xA0 -> UNDEFINED - '\ufffe', // 0xA1 -> UNDEFINED - '\ufffe', // 0xA2 -> UNDEFINED - '\ufffe', // 0xA3 -> UNDEFINED - '\ufffe', // 0xA4 -> UNDEFINED - '\ufffe', // 0xA5 -> UNDEFINED - '\ufffe', // 0xA6 -> UNDEFINED - '\ufffe', // 0xA7 -> UNDEFINED - '\ufffe', // 0xA8 -> UNDEFINED - '\ufffe', // 0xA9 -> UNDEFINED - '\ufffe', // 0xAA -> UNDEFINED - '\ufffe', // 0xAB -> UNDEFINED - '\ufffe', // 0xAC -> UNDEFINED - '\ufffe', // 0xAD -> UNDEFINED - '\ufffe', // 0xAE -> UNDEFINED - '\ufffe', // 0xAF -> UNDEFINED - '\ufffe', // 0xB0 -> UNDEFINED - '\ufffe', // 0xB1 -> UNDEFINED - '\ufffe', // 0xB2 -> UNDEFINED - '\ufffe', // 0xB3 -> UNDEFINED - '\ufffe', // 0xB4 -> UNDEFINED - '\ufffe', // 0xB5 -> UNDEFINED - '\ufffe', // 0xB6 -> UNDEFINED - '\ufffe', // 0xB7 -> UNDEFINED - '\ufffe', // 0xB8 -> UNDEFINED - '\ufffe', // 0xB9 -> UNDEFINED - '\ufffe', // 0xBA -> UNDEFINED - '\ufffe', // 0xBB -> UNDEFINED - '\ufffe', // 0xBC -> UNDEFINED - '\ufffe', // 0xBD -> UNDEFINED - '\ufffe', // 0xBE -> UNDEFINED - '\ufffe', // 0xBF -> UNDEFINED - '\ufffe', // 0xC0 -> UNDEFINED - '\ufffe', // 0xC1 -> UNDEFINED - '\ufffe', // 0xC2 -> UNDEFINED - '\ufffe', // 0xC3 -> UNDEFINED - '\ufffe', // 0xC4 -> UNDEFINED - '\ufffe', // 0xC5 -> UNDEFINED - '\ufffe', // 0xC6 -> UNDEFINED - '\ufffe', // 0xC7 -> UNDEFINED - '\ufffe', // 0xC8 -> UNDEFINED - '\ufffe', // 0xC9 -> UNDEFINED - '\ufffe', // 0xCA -> UNDEFINED - '\ufffe', // 0xCB -> UNDEFINED - '\ufffe', // 0xCC -> UNDEFINED - '\ufffe', // 0xCD -> UNDEFINED - '\ufffe', // 0xCE -> UNDEFINED - '\ufffe', // 0xCF -> UNDEFINED - '\ufffe', // 0xD0 -> UNDEFINED - '\ufffe', // 0xD1 -> UNDEFINED - '\ufffe', // 0xD2 -> UNDEFINED - '\ufffe', // 0xD3 -> UNDEFINED - '\ufffe', // 0xD4 -> UNDEFINED - '\ufffe', // 0xD5 -> UNDEFINED - '\ufffe', // 0xD6 -> UNDEFINED - '\ufffe', // 0xD7 -> UNDEFINED - '\ufffe', // 0xD8 -> UNDEFINED - '\ufffe', // 0xD9 -> UNDEFINED - '\ufffe', // 0xDA -> UNDEFINED - '\ufffe', // 0xDB -> UNDEFINED - '\ufffe', // 0xDC -> UNDEFINED - '\ufffe', // 0xDD -> UNDEFINED - '\ufffe', // 0xDE -> UNDEFINED - '\ufffe', // 0xDF -> UNDEFINED - '\ufffe', // 0xE0 -> UNDEFINED - '\ufffe', // 0xE1 -> UNDEFINED - '\ufffe', // 0xE2 -> UNDEFINED - '\ufffe', // 0xE3 -> UNDEFINED - '\ufffe', // 0xE4 -> UNDEFINED - '\ufffe', // 0xE5 -> UNDEFINED - '\ufffe', // 0xE6 -> UNDEFINED - '\ufffe', // 0xE7 -> UNDEFINED - '\ufffe', // 0xE8 -> UNDEFINED - '\ufffe', // 0xE9 -> UNDEFINED - '\ufffe', // 0xEA -> UNDEFINED - '\ufffe', // 0xEB -> UNDEFINED - '\ufffe', // 0xEC -> UNDEFINED - '\ufffe', // 0xED -> UNDEFINED - '\ufffe', // 0xEE -> UNDEFINED - '\ufffe', // 0xEF -> UNDEFINED - '\ufffe', // 0xF0 -> UNDEFINED - '\ufffe', // 0xF1 -> UNDEFINED - '\ufffe', // 0xF2 -> UNDEFINED - '\ufffe', // 0xF3 -> UNDEFINED - '\ufffe', // 0xF4 -> UNDEFINED - '\ufffe', // 0xF5 -> UNDEFINED - '\ufffe', // 0xF6 -> UNDEFINED - '\ufffe', // 0xF7 -> UNDEFINED - '\ufffe', // 0xF8 -> UNDEFINED - '\ufffe', // 0xF9 -> UNDEFINED - '\ufffe', // 0xFA -> UNDEFINED - '\ufffe', // 0xFB -> UNDEFINED - '\ufffe', // 0xFC -> UNDEFINED - '\ufffe', // 0xFD -> UNDEFINED - '\ufffe', // 0xFE -> UNDEFINED - '\ufffe' // 0xFF -> UNDEFINED - ) - - // encoding: from unicode to Petscii/Screencodes (0-255) - private val encodingPetsciiLowercase = decodingPetsciiLowercase.withIndex().associate{it.value to it.index} - private val encodingPetsciiUppercase = decodingPetsciiUppercase.withIndex().associate{it.value to it.index} - private val encodingScreencodeLowercase = decodingScreencodeLowercase.withIndex().associate{it.value to it.index} - private val encodingScreencodeUppercase = decodingScreencodeUppercase.withIndex().associate{it.value to it.index} - - private fun replaceSpecial(chr: Char): Char = - // characters often used in C like source code can be translated with a little bit of fantasy: - when(chr) { - '^' -> '↑' - '_' -> '▁' - '{' -> '┤' - '}' -> '├' - '|' -> '│' - '\\' -> '╲' - else -> chr - } - - fun encodePetscii(text: String, lowercase: Boolean = false): List { - fun encodeChar(chr3: Char, lowercase: Boolean): Short { - val chr = replaceSpecial(chr3) - val screencode = if(lowercase) encodingPetsciiLowercase[chr] else encodingPetsciiUppercase[chr] - return screencode?.toShort() ?: when (chr) { - '\u0000' -> 0.toShort() - in '\u8000'..'\u80ff' -> { - // special case: take the lower 8 bit hex value directly - (chr.code - 0x8000).toShort() - } - else -> { - val case = if (lowercase) "lower" else "upper" - throw CharConversionException("no ${case}Petscii character for '${escape(chr.toString())}' (${chr.code})") - } - } - } - - return text.map{ - try { - encodeChar(it, lowercase) - } catch (x: CharConversionException) { - encodeChar(it, !lowercase) - } - } - } - - fun decodePetscii(petscii: Iterable, lowercase: Boolean = false): String { - return petscii.map { - val code = it.toInt() - try { - if(lowercase) decodingPetsciiLowercase[code] else decodingPetsciiUppercase[code] - } catch(x: CharConversionException) { - if(lowercase) decodingPetsciiUppercase[code] else decodingPetsciiLowercase[code] - } - }.joinToString("") - } - - fun encodeScreencode(text: String, lowercase: Boolean = false): List { - fun encodeChar(chr3: Char, lowercase: Boolean): Short { - val chr = replaceSpecial(chr3) - val screencode = if(lowercase) encodingScreencodeLowercase[chr] else encodingScreencodeUppercase[chr] - return screencode?.toShort() ?: when (chr) { - '\u0000' -> 0.toShort() - in '\u8000'..'\u80ff' -> { - // special case: take the lower 8 bit hex value directly - (chr.code - 0x8000).toShort() - } - else -> { - val case = if (lowercase) "lower" else "upper" - throw CharConversionException("no ${case}Screencode character for '${escape(chr.toString())}' (${chr.code})") - } - } - } - - return text.map{ - try { - encodeChar(it, lowercase) - } catch (x: CharConversionException) { - encodeChar(it, !lowercase) - } - } - } - - fun decodeScreencode(screencode: Iterable, lowercase: Boolean = false): String { - return screencode.map { - val code = it.toInt() - try { - if (lowercase) decodingScreencodeLowercase[code] else decodingScreencodeUppercase[code] - } catch (x: CharConversionException) { - if (lowercase) decodingScreencodeUppercase[code] else decodingScreencodeLowercase[code] - } - }.joinToString("") - } - - fun petscii2scr(petscii_code: Short, inverseVideo: Boolean): Short { - val code = when { - petscii_code <= 0x1f -> petscii_code + 128 - petscii_code <= 0x3f -> petscii_code.toInt() - petscii_code <= 0x5f -> petscii_code - 64 - petscii_code <= 0x7f -> petscii_code - 32 - petscii_code <= 0x9f -> petscii_code + 64 - petscii_code <= 0xbf -> petscii_code - 64 - petscii_code <= 0xfe -> petscii_code - 128 - petscii_code == 255.toShort() -> 95 - else -> throw CharConversionException("petscii code out of range") - } - if(inverseVideo) - return (code or 0x80).toShort() - return code.toShort() - } - - fun scr2petscii(screencode: Short): Short { - val petscii = when { - screencode <= 0x1f -> screencode + 64 - screencode <= 0x3f -> screencode.toInt() - screencode <= 0x5d -> screencode +123 - screencode == 0x5e.toShort() -> 255 - screencode == 0x5f.toShort() -> 223 - screencode <= 0x7f -> screencode + 64 - screencode <= 0xbf -> screencode - 128 - screencode <= 0xfe -> screencode - 64 - screencode == 255.toShort() -> 191 - else -> throw CharConversionException("screencode out of range") - } - return petscii.toShort() - } -} diff --git a/compilerAst/src/prog8/parser/PetsciiEncoding.kt b/compilerAst/src/prog8/parser/PetsciiEncoding.kt deleted file mode 100644 index 9817003bd..000000000 --- a/compilerAst/src/prog8/parser/PetsciiEncoding.kt +++ /dev/null @@ -1,24 +0,0 @@ -package prog8.parser - -import prog8.ast.IStringEncoding -import java.io.CharConversionException - - -/** - * TODO: remove once [IStringEncoding] has been moved to compiler module - */ -object PetsciiEncoding : IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean) = - try { - if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true) - } catch (x: CharConversionException) { - throw CharConversionException("can't convert string to target machine's char encoding: ${x.message}") - } - - override fun decodeString(bytes: List, altEncoding: Boolean) = - try { - if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true) - } catch (x: CharConversionException) { - throw CharConversionException("can't decode string: ${x.message}") - } -} diff --git a/compilerAst/src/prog8/parser/Prog8Parser.kt b/compilerAst/src/prog8/parser/Prog8Parser.kt index 4d080a8ea..1f783fada 100644 --- a/compilerAst/src/prog8/parser/Prog8Parser.kt +++ b/compilerAst/src/prog8/parser/Prog8Parser.kt @@ -37,7 +37,7 @@ object Prog8Parser { // .linkParents called in ParsedModule.add parseTree.directive().forEach { module.add(it.toAst()) } // TODO: remove Encoding - parseTree.block().forEach { module.add(it.toAst(module.isLibrary(), PetsciiEncoding)) } + parseTree.block().forEach { module.add(it.toAst(module.isLibrary())) } return module } diff --git a/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt b/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt deleted file mode 100644 index 57f48871e..000000000 --- a/compilerAst/src/prog8/parser/ThrowTodoEncoding.kt +++ /dev/null @@ -1,16 +0,0 @@ -package prog8.parser - -import prog8.ast.IStringEncoding - -/** - * TODO: remove once [IStringEncoding] has been moved to compiler module - */ -object ThrowTodoEncoding: IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("move StringEncoding out of compilerAst") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("move StringEncoding out of compilerAst") - } -} diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/Helpers.kt index a49aed04a..1a59142a9 100644 --- a/compilerAst/test/Helpers.kt +++ b/compilerAst/test/Helpers.kt @@ -5,7 +5,6 @@ import kotlin.io.path.* import prog8.ast.IBuiltinFunctions import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding import prog8.ast.base.DataType import prog8.ast.base.Position import prog8.ast.expressions.Expression @@ -46,16 +45,6 @@ fun sanityCheckDirectories(workingDirName: String? = null) { } - val DummyEncoding = object : IStringEncoding { - override fun encodeString(str: String, altEncoding: Boolean): List { - TODO("Not yet implemented") - } - - override fun decodeString(bytes: List, altEncoding: Boolean): String { - TODO("Not yet implemented") - } -} - val DummyFunctions = object : IBuiltinFunctions { override val names: Set = emptySet() override val purefunctionNames: Set = emptySet() diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 68ebb41e4..e28a85bb0 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -19,7 +19,7 @@ class TestModuleImporter { fun testImportModuleWithNonExistingPath() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val srcPath = fixturesDir.resolve("i_do_not_exist") assumeNotExists(srcPath) @@ -31,7 +31,7 @@ class TestModuleImporter { fun testImportModuleWithDirectoryPath() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val srcPath = fixturesDir assumeDirectory(srcPath) @@ -46,7 +46,7 @@ class TestModuleImporter { fun testImportModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val filename = "file_with_syntax_error.p8" val path = fixturesDir.resolve(filename) @@ -67,7 +67,7 @@ class TestModuleImporter { fun testImportModuleWithImportingModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") val imported = fixturesDir.resolve("file_with_syntax_error.p8") @@ -92,7 +92,7 @@ class TestModuleImporter { fun testImportLibraryModuleWithNonExistingName() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val filenameNoExt = "i_do_not_exist" val filenameWithExt = filenameNoExt + ".p8" val srcPathNoExt = fixturesDir.resolve(filenameNoExt) @@ -108,7 +108,7 @@ class TestModuleImporter { fun testImportLibraryModuleWithSyntaxError() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val srcPath = fixturesDir.resolve("file_with_syntax_error.p8") assumeReadableFile(srcPath) @@ -130,7 +130,7 @@ class TestModuleImporter { fun testImportLibraryModuleWithImportingBadModule() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, DummyEncoding, "blah", listOf(searchIn)) + val importer = ModuleImporter(program, "blah", listOf(searchIn)) val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") val imported = fixturesDir.resolve("file_with_syntax_error.p8") From 4c615e4fac2d89d79ca10b9e12128ee801960198 Mon Sep 17 00:00:00 2001 From: meisl Date: Fri, 30 Jul 2021 17:19:44 +0200 Subject: [PATCH 41/68] * solve problem re shared test helpers: a) don't use the same file name (results in same JVM class name) & b) tell gradle about it (put them in extra dir(s) test/helpers/ and add this to test source set) --- compiler/build.gradle | 3 +- .../Helpers@compiler.kt} | 54 ------------------- .../Helpers@compilerAst.kt} | 0 3 files changed, 2 insertions(+), 55 deletions(-) rename compiler/test/{Helpers.kt => helpers/Helpers@compiler.kt} (61%) rename compilerAst/test/{Helpers.kt => helpers/Helpers@compilerAst.kt} (100%) diff --git a/compiler/build.gradle b/compiler/build.gradle index 037f8515a..15093c8a8 100644 --- a/compiler/build.gradle +++ b/compiler/build.gradle @@ -70,7 +70,8 @@ sourceSets { } test { java { - srcDirs = ["${project.projectDir}/test"] + srcDir "${project.projectDir}/test" + srcDir "${project(':compilerAst').projectDir}/test/helpers" } } } diff --git a/compiler/test/Helpers.kt b/compiler/test/helpers/Helpers@compiler.kt similarity index 61% rename from compiler/test/Helpers.kt rename to compiler/test/helpers/Helpers@compiler.kt index 48f745b2d..85c1f5934 100644 --- a/compiler/test/Helpers.kt +++ b/compiler/test/helpers/Helpers@compiler.kt @@ -15,8 +15,6 @@ import prog8.compiler.CompilationResult import prog8.compiler.compileProgram import prog8.compiler.target.ICompilationTarget -// TODO: find a way to share with compilerAst/test/Helpers.kt, while still being able to amend it (-> compileFile(..)) - internal fun CompilationResult.assertSuccess(description: String = ""): CompilationResult { assertTrue(success, "expected successful compilation but failed $description") return this @@ -68,39 +66,6 @@ internal fun compileText( } - -val workingDir : Path = Path("").absolute() // Note: Path(".") does NOT work..! -val fixturesDir : Path = workingDir.resolve("test/fixtures") -val resourcesDir : Path = workingDir.resolve("res") -val outputDir : Path = workingDir.resolve("build/tmp/test") - -fun assumeReadable(path: Path) { - assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") -} - -fun assumeReadableFile(path: Path) { - assumeReadable(path) - assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") -} - -fun assumeDirectory(path: Path) { - assertTrue(path.isDirectory(), "sanity check; should be directory: $path") -} - -fun assumeNotExists(path: Path) { - assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") -} - -fun sanityCheckDirectories(workingDirName: String? = null) { - if (workingDirName != null) - assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") - assumeDirectory(workingDir) - assumeDirectory(fixturesDir) - assumeDirectory(resourcesDir) - assumeDirectory(outputDir) -} - - fun mapCombinations(dim1: Iterable, dim2: Iterable, combine2: (A, B) -> R) = sequence { for (a in dim1) @@ -124,22 +89,3 @@ fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: for (d in dim4) yield(combine4(a, b, c, d)) }.toList() - - -val DummyFunctions = object : IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue( - name: String, - args: List, - position: Position, - memsizer: IMemSizer - ): NumericLiteralValue? = null - - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() -} - -val DummyMemsizer = object : IMemSizer { - override fun memorySize(dt: DataType): Int = 0 -} - diff --git a/compilerAst/test/Helpers.kt b/compilerAst/test/helpers/Helpers@compilerAst.kt similarity index 100% rename from compilerAst/test/Helpers.kt rename to compilerAst/test/helpers/Helpers@compilerAst.kt From e1026584c84516d4a9e24cb05be75b5ec9676bfc Mon Sep 17 00:00:00 2001 From: meisl Date: Fri, 30 Jul 2021 17:39:43 +0200 Subject: [PATCH 42/68] * split up test helpers into separate files, move mapCombinations(..) down to compilerAst/test/helpers since they're generic and don't depend on compiler --- .../{Helpers@compiler.kt => compileXyz.kt} | 33 +------------------ compilerAst/test/helpers/DummyFunctions.kt | 21 ++++++++++++ compilerAst/test/helpers/DummyMemsizer.kt | 8 +++++ compilerAst/test/helpers/mapCombinations.kt | 25 ++++++++++++++ .../{Helpers@compilerAst.kt => paths.kt} | 26 --------------- 5 files changed, 55 insertions(+), 58 deletions(-) rename compiler/test/helpers/{Helpers@compiler.kt => compileXyz.kt} (61%) create mode 100644 compilerAst/test/helpers/DummyFunctions.kt create mode 100644 compilerAst/test/helpers/DummyMemsizer.kt create mode 100644 compilerAst/test/helpers/mapCombinations.kt rename compilerAst/test/helpers/{Helpers@compilerAst.kt => paths.kt} (59%) diff --git a/compiler/test/helpers/Helpers@compiler.kt b/compiler/test/helpers/compileXyz.kt similarity index 61% rename from compiler/test/helpers/Helpers@compiler.kt rename to compiler/test/helpers/compileXyz.kt index 85c1f5934..1862b7e01 100644 --- a/compiler/test/helpers/Helpers@compiler.kt +++ b/compiler/test/helpers/compileXyz.kt @@ -4,17 +4,11 @@ import kotlin.test.* import kotlin.io.path.* import java.nio.file.Path -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import prog8.compiler.CompilationResult import prog8.compiler.compileProgram import prog8.compiler.target.ICompilationTarget + internal fun CompilationResult.assertSuccess(description: String = ""): CompilationResult { assertTrue(success, "expected successful compilation but failed $description") return this @@ -64,28 +58,3 @@ internal fun compileText( filePath.toFile().writeText(sourceText) return compileFile(platform, optimize, filePath.parent, filePath.name) } - - -fun mapCombinations(dim1: Iterable, dim2: Iterable, combine2: (A, B) -> R) = - sequence { - for (a in dim1) - for (b in dim2) - yield(combine2(a, b)) - }.toList() - -fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, combine3: (A, B, C) -> R) = - sequence { - for (a in dim1) - for (b in dim2) - for (c in dim3) - yield(combine3(a, b, c)) - }.toList() - -fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, dim4: Iterable, combine4: (A, B, C, D) -> R) = - sequence { - for (a in dim1) - for (b in dim2) - for (c in dim3) - for (d in dim4) - yield(combine4(a, b, c, d)) - }.toList() diff --git a/compilerAst/test/helpers/DummyFunctions.kt b/compilerAst/test/helpers/DummyFunctions.kt new file mode 100644 index 000000000..e212ea9ea --- /dev/null +++ b/compilerAst/test/helpers/DummyFunctions.kt @@ -0,0 +1,21 @@ +package prog8tests.helpers + +import prog8.ast.IBuiltinFunctions +import prog8.ast.IMemSizer +import prog8.ast.base.Position +import prog8.ast.expressions.Expression +import prog8.ast.expressions.InferredTypes +import prog8.ast.expressions.NumericLiteralValue + +val DummyFunctions = object : IBuiltinFunctions { + override val names: Set = emptySet() + override val purefunctionNames: Set = emptySet() + override fun constValue( + name: String, + args: List, + position: Position, + memsizer: IMemSizer + ): NumericLiteralValue? = null + + override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() +} \ No newline at end of file diff --git a/compilerAst/test/helpers/DummyMemsizer.kt b/compilerAst/test/helpers/DummyMemsizer.kt new file mode 100644 index 000000000..3d226437d --- /dev/null +++ b/compilerAst/test/helpers/DummyMemsizer.kt @@ -0,0 +1,8 @@ +package prog8tests.helpers + +import prog8.ast.IMemSizer +import prog8.ast.base.DataType + +val DummyMemsizer = object : IMemSizer { + override fun memorySize(dt: DataType): Int = 0 +} \ No newline at end of file diff --git a/compilerAst/test/helpers/mapCombinations.kt b/compilerAst/test/helpers/mapCombinations.kt new file mode 100644 index 000000000..e96734cb1 --- /dev/null +++ b/compilerAst/test/helpers/mapCombinations.kt @@ -0,0 +1,25 @@ +package prog8tests.helpers + +fun mapCombinations(dim1: Iterable, dim2: Iterable, combine2: (A, B) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + yield(combine2(a, b)) + }.toList() + +fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, combine3: (A, B, C) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + for (c in dim3) + yield(combine3(a, b, c)) + }.toList() + +fun mapCombinations(dim1: Iterable, dim2: Iterable, dim3: Iterable, dim4: Iterable, combine4: (A, B, C, D) -> R) = + sequence { + for (a in dim1) + for (b in dim2) + for (c in dim3) + for (d in dim4) + yield(combine4(a, b, c, d)) + }.toList() \ No newline at end of file diff --git a/compilerAst/test/helpers/Helpers@compilerAst.kt b/compilerAst/test/helpers/paths.kt similarity index 59% rename from compilerAst/test/helpers/Helpers@compilerAst.kt rename to compilerAst/test/helpers/paths.kt index 1a59142a9..65c5de9fd 100644 --- a/compilerAst/test/helpers/Helpers@compilerAst.kt +++ b/compilerAst/test/helpers/paths.kt @@ -3,13 +3,6 @@ package prog8tests.helpers import kotlin.test.* import kotlin.io.path.* -import prog8.ast.IBuiltinFunctions -import prog8.ast.IMemSizer -import prog8.ast.base.DataType -import prog8.ast.base.Position -import prog8.ast.expressions.Expression -import prog8.ast.expressions.InferredTypes -import prog8.ast.expressions.NumericLiteralValue import java.nio.file.Path @@ -43,22 +36,3 @@ fun sanityCheckDirectories(workingDirName: String? = null) { assumeDirectory(resourcesDir) assumeDirectory(outputDir) } - - -val DummyFunctions = object : IBuiltinFunctions { - override val names: Set = emptySet() - override val purefunctionNames: Set = emptySet() - override fun constValue( - name: String, - args: List, - position: Position, - memsizer: IMemSizer - ): NumericLiteralValue? = null - - override fun returnType(name: String, args: MutableList) = InferredTypes.InferredType.unknown() -} - -val DummyMemsizer = object : IMemSizer { - override fun memorySize(dt: DataType): Int = 0 -} - From ed061b362b01c1a289317e1d4966d311f0476183 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 18 Jul 2021 08:50:12 +0200 Subject: [PATCH 43/68] * #53 step6: move IStringEncoding to prog8.compiler (package as well as module) --- .../ast => compiler/src/prog8/compiler}/IStringEncoding.kt | 2 +- compiler/src/prog8/compiler/astprocessing/AstExtensions.kt | 2 +- compiler/src/prog8/compiler/target/ICompilationTarget.kt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename {compilerAst/src/prog8/ast => compiler/src/prog8/compiler}/IStringEncoding.kt (88%) diff --git a/compilerAst/src/prog8/ast/IStringEncoding.kt b/compiler/src/prog8/compiler/IStringEncoding.kt similarity index 88% rename from compilerAst/src/prog8/ast/IStringEncoding.kt rename to compiler/src/prog8/compiler/IStringEncoding.kt index 8296d2988..a69e9ba62 100644 --- a/compilerAst/src/prog8/ast/IStringEncoding.kt +++ b/compiler/src/prog8/compiler/IStringEncoding.kt @@ -1,4 +1,4 @@ -package prog8.ast +package prog8.compiler interface IStringEncoding { fun encodeString(str: String, altEncoding: Boolean): List diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index 3eb3760a6..8f9ed35fe 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -1,6 +1,6 @@ package prog8.compiler.astprocessing -import prog8.ast.IStringEncoding +import prog8.compiler.IStringEncoding import prog8.ast.Node import prog8.ast.Program import prog8.ast.base.DataType diff --git a/compiler/src/prog8/compiler/target/ICompilationTarget.kt b/compiler/src/prog8/compiler/target/ICompilationTarget.kt index 6f71ccd3c..d232f89f9 100644 --- a/compiler/src/prog8/compiler/target/ICompilationTarget.kt +++ b/compiler/src/prog8/compiler/target/ICompilationTarget.kt @@ -1,7 +1,7 @@ package prog8.compiler.target import prog8.ast.IMemSizer -import prog8.ast.IStringEncoding +import prog8.compiler.IStringEncoding import prog8.ast.Program import prog8.ast.base.* import prog8.ast.expressions.IdentifierReference From 1b451180c134d68fe79fec7acd9eee207b2d2eb8 Mon Sep 17 00:00:00 2001 From: meisl Date: Sat, 31 Jul 2021 14:44:02 +0200 Subject: [PATCH 44/68] * test helpers assumeXyz (helpers/paths.kt) return the resulting path (unless they fail, of course); test directories are checked automatically at init, so no sanityCheckDirectories is needed anymore --- compiler/test/TestCompilerOnCharLit.kt | 5 - compiler/test/TestCompilerOnExamples.kt | 10 +- .../test/TestCompilerOnImportsAndIncludes.kt | 33 +- compiler/test/TestCompilerOnRanges.kt | 9 +- compilerAst/test/TestModuleImporter.kt | 32 +- compilerAst/test/TestProg8Parser.kt | 29 +- compilerAst/test/TestSourceCode.kt | 69 ++-- compilerAst/test/helpers/paths.kt | 63 ++-- compilerAst/test/helpers_pathsTests.kt | 311 ++++++++++++++++++ 9 files changed, 404 insertions(+), 157 deletions(-) create mode 100644 compilerAst/test/helpers_pathsTests.kt diff --git a/compiler/test/TestCompilerOnCharLit.kt b/compiler/test/TestCompilerOnCharLit.kt index 08d50c272..5f82311bc 100644 --- a/compiler/test/TestCompilerOnCharLit.kt +++ b/compiler/test/TestCompilerOnCharLit.kt @@ -24,11 +24,6 @@ import prog8.compiler.target.Cx16Target @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnCharLit { - @BeforeAll - fun setUp() { - sanityCheckDirectories("compiler") - } - @Test fun testCharLitAsRomsubArg() { val platform = Cx16Target diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index a98e2bade..e4d9a2254 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -23,20 +23,14 @@ import prog8.compiler.target.ICompilationTarget //@Disabled("to save some time") @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnExamples { - private val examplesDir = workingDir.resolve("../examples") - - @BeforeAll - fun setUp() { - sanityCheckDirectories("compiler") - assumeDirectory(examplesDir) - } + private val examplesDir = assumeDirectory(workingDir, "../examples") // TODO: make assembly stage testable - in case of failure (eg of 64tass) it Process.exit s private fun makeDynamicCompilerTest(name: String, platform: ICompilationTarget, optimize: Boolean) : DynamicTest { val searchIn = mutableListOf(examplesDir) if (platform == Cx16Target) { - searchIn.add(0, examplesDir.resolve("cx16")) + searchIn.add(0, assumeDirectory(examplesDir, "cx16")) } val filepath = searchIn.map { it.resolve("$name.p8") }.first { it.exists() } val displayName = "${examplesDir.relativize(filepath)}: ${platform.name}, optimize=$optimize" diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index d166f92d3..f08259c11 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -26,17 +26,10 @@ import prog8.compiler.target.Cx16Target @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnImportsAndIncludes { - @BeforeAll - fun setUp() { - sanityCheckDirectories("compiler") - } - @Test fun testImportFromSameFolder() { - val filepath = fixturesDir.resolve("importFromSameFolder.p8") - val imported = fixturesDir.resolve("foo_bar.p8") - assumeReadableFile(filepath) - assumeReadableFile(imported) + val filepath = assumeReadableFile(fixturesDir, "importFromSameFolder.p8") + assumeReadableFile(fixturesDir, "foo_bar.p8") val platform = Cx16Target val result = compileFile(platform, false, fixturesDir, filepath.name) @@ -57,10 +50,8 @@ class TestCompilerOnImportsAndIncludes { @Test fun testAsmIncludeFromSameFolder() { - val filepath = fixturesDir.resolve("asmIncludeFromSameFolder.p8") - val included = fixturesDir.resolve("foo_bar.asm") - assumeReadableFile(filepath) - assumeReadableFile(included) + val filepath = assumeReadableFile(fixturesDir, "asmIncludeFromSameFolder.p8") + assumeReadableFile(fixturesDir,"foo_bar.asm") val platform = Cx16Target val result = compileFile(platform, false, fixturesDir, filepath.name) @@ -84,10 +75,8 @@ class TestCompilerOnImportsAndIncludes { @Test fun testAsmbinaryDirectiveWithNonExistingFile() { - val p8Path = fixturesDir.resolve("asmbinaryNonExisting.p8") - val binPath = fixturesDir.resolve("i_do_not_exist.bin") - assumeReadableFile(p8Path) - assumeNotExists(binPath) + val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonExisting.p8") + assumeNotExists(fixturesDir,"i_do_not_exist.bin") compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) .assertFailure() @@ -95,10 +84,8 @@ class TestCompilerOnImportsAndIncludes { @Test fun testAsmbinaryDirectiveWithNonReadableFile() { - val p8Path = fixturesDir.resolve("asmbinaryNonReadable.p8") - val binPath = fixturesDir.resolve("subFolder") - assumeReadableFile(p8Path) - assumeDirectory(binPath) + val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonReadable.p8") + assumeDirectory(fixturesDir, "subFolder") compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) .assertFailure() @@ -111,8 +98,8 @@ class TestCompilerOnImportsAndIncludes { Triple("sub", "asmBinaryFromSubFolder.p8", "subFolder/do_nothing2.bin"), ).map { val (where, p8Str, binStr) = it - val p8Path = fixturesDir.resolve(p8Str) - val binPath = fixturesDir.resolve(binStr) + val p8Path = fixturesDir.div(p8Str) // sanity check below, *inside dynamicTest* + val binPath = fixturesDir.div(binStr) val displayName = "%asmbinary from ${where}folder" dynamicTest(displayName) { assumeReadableFile(p8Path) diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index c6c36cc78..5f5fafdcb 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -28,11 +28,6 @@ import prog8.compiler.target.Cx16Target @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnRanges { - @BeforeAll - fun setUp() { - sanityCheckDirectories("compiler") - } - @Test fun testUByteArrayInitializerWithRange_char_to_char() { val platform = Cx16Target @@ -91,7 +86,7 @@ class TestCompilerOnRanges { assertEquals(expectedEnd - expectedStart + 1, rhsValues.size, "rangeExpr.size()") } - inline fun Subroutine.decl(varName: String): VarDecl { + fun Subroutine.decl(varName: String): VarDecl { return statements.filterIsInstance() .first { it.name == varName } } @@ -231,7 +226,7 @@ class TestCompilerOnRanges { @Test fun testForLoopWithRange_str_downto_str() { - val result = compileText(Cx16Target, true, """ + compileText(Cx16Target, true, """ main { sub start() { ubyte i diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index e28a85bb0..0466e8123 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -20,9 +20,7 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - - val srcPath = fixturesDir.resolve("i_do_not_exist") - assumeNotExists(srcPath) + val srcPath = assumeNotExists(fixturesDir, "i_do_not_exist") assertFailsWith { importer.importModule(srcPath) } } @@ -33,8 +31,7 @@ class TestModuleImporter { val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = fixturesDir - assumeDirectory(srcPath) + val srcPath = assumeDirectory(fixturesDir) // fn importModule(Path) used to check *.isReadable()*, but NOT .isRegularFile(): assumeReadable(srcPath) @@ -49,7 +46,7 @@ class TestModuleImporter { val importer = ModuleImporter(program, "blah", listOf(searchIn)) val filename = "file_with_syntax_error.p8" - val path = fixturesDir.resolve(filename) + val path = assumeReadableFile(fixturesDir, filename) val act = { importer.importModule(path) } assertFailsWith { act() } @@ -69,10 +66,8 @@ class TestModuleImporter { val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") - val imported = fixturesDir.resolve("file_with_syntax_error.p8") - assumeReadableFile(importing) - assumeReadableFile(imported) + val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") + val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") val act = { importer.importModule(importing) } @@ -93,12 +88,8 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val filenameNoExt = "i_do_not_exist" - val filenameWithExt = filenameNoExt + ".p8" - val srcPathNoExt = fixturesDir.resolve(filenameNoExt) - val srcPathWithExt = fixturesDir.resolve(filenameWithExt) - assumeNotExists(srcPathNoExt) - assumeNotExists(srcPathWithExt) + val filenameNoExt = assumeNotExists(fixturesDir, "i_do_not_exist").name + val filenameWithExt = assumeNotExists(fixturesDir, "i_do_not_exist.p8").name assertFailsWith { importer.importLibraryModule(filenameNoExt) } assertFailsWith { importer.importLibraryModule(filenameWithExt) } @@ -109,8 +100,7 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = fixturesDir.resolve("file_with_syntax_error.p8") - assumeReadableFile(srcPath) + val srcPath = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") val act = { importer.importLibraryModule(srcPath.nameWithoutExtension) } @@ -132,10 +122,8 @@ class TestModuleImporter { val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = fixturesDir.resolve("import_file_with_syntax_error.p8") - val imported = fixturesDir.resolve("file_with_syntax_error.p8") - assumeReadableFile(importing) - assumeReadableFile(imported) + val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") + val imported = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") val act = { importer.importLibraryModule(importing.nameWithoutExtension) } diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 622198561..55e7da561 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -1,11 +1,10 @@ package prog8tests +import prog8tests.helpers.* import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.Test import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.BeforeAll import kotlin.test.* -import prog8tests.helpers.* import kotlin.io.path.* import prog8.parser.ParseError @@ -20,11 +19,6 @@ import prog8.ast.expressions.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestProg8Parser { - @BeforeAll - fun setUp() { - sanityCheckDirectories("compilerAst") - } - @Test fun testModuleSourceNeedNotEndWithNewline() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) @@ -166,11 +160,8 @@ class TestProg8Parser { @Test fun parseModuleShouldNotLookAtImports() { - val importedNoExt = fixturesDir.resolve("i_do_not_exist") - val importedWithExt = fixturesDir.resolve("i_do_not_exist.p8") - assumeNotExists(importedNoExt) - assumeNotExists(importedWithExt) - + val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") + assumeNotExists(fixturesDir, "i_do_not_exist.p8") val text = "%import ${importedNoExt.name}" val module = parseModule(SourceCode.of(text)) @@ -186,9 +177,7 @@ class TestProg8Parser { @Test fun testParseModuleWithEmptyFile() { - val path = fixturesDir.resolve("empty.p8") - assumeReadableFile(path) - + val path = assumeReadableFile(fixturesDir,"empty.p8") val module = parseModule(SourceCode.fromPath(path)) assertEquals(0, module.statements.size) } @@ -207,10 +196,8 @@ class TestProg8Parser { @Test fun testModuleNameForSourceFromPath() { - val path = fixturesDir.resolve("simple_main.p8") - + val path = assumeReadableFile(fixturesDir,"simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) - assertEquals(path.nameWithoutExtension, module.name) } @@ -253,7 +240,7 @@ class TestProg8Parser { @Test fun testErrorLocationForSourceFromPath() { - val path = fixturesDir.resolve("file_with_syntax_error.p8") + val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") assertFailsWith { parseModule(SourceCode.fromPath(path)) } try { @@ -275,7 +262,7 @@ class TestProg8Parser { @Test fun testModulePositionForSourceFromPath() { - val path = fixturesDir.resolve("simple_main.p8") + val path = assumeReadableFile(fixturesDir,"simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong @@ -283,7 +270,7 @@ class TestProg8Parser { @Test fun testInnerNodePositionsForSourceFromPath() { - val path = fixturesDir.resolve("simple_main.p8") + val path = assumeReadableFile(fixturesDir,"simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) val mpf = module.position.file diff --git a/compilerAst/test/TestSourceCode.kt b/compilerAst/test/TestSourceCode.kt index 571f23eb2..13ce07fe1 100644 --- a/compilerAst/test/TestSourceCode.kt +++ b/compilerAst/test/TestSourceCode.kt @@ -1,11 +1,10 @@ package prog8tests +import prog8tests.helpers.* import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.Test -import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.* import kotlin.test.* -import prog8tests.helpers.* import kotlin.io.path.* import prog8.parser.SourceCode @@ -14,11 +13,6 @@ import prog8.parser.SourceCode @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestSourceCode { - @BeforeAll - fun setUp() { - sanityCheckDirectories("compilerAst") - } - @Test fun testFactoryMethod_Of() { val text = """ @@ -34,16 +28,14 @@ class TestSourceCode { @Test fun testFromPathWithNonExistingPath() { val filename = "i_do_not_exist.p8" - val path = fixturesDir.resolve(filename) - assumeNotExists(path) + val path = assumeNotExists(fixturesDir, filename) assertFailsWith { SourceCode.fromPath(path) } } @Test fun testFromPathWithMissingExtension_p8() { - val pathWithoutExt = fixturesDir.resolve("simple_main") - val pathWithExt = Path(pathWithoutExt.toString() + ".p8") - assumeReadableFile(pathWithExt) + val pathWithoutExt = assumeNotExists(fixturesDir,"simple_main") + assumeReadableFile(fixturesDir,"simple_main.p8") assertFailsWith { SourceCode.fromPath(pathWithoutExt) } } @@ -55,90 +47,75 @@ class TestSourceCode { @Test fun testFromPathWithExistingPath() { val filename = "simple_main.p8" - val path = fixturesDir.resolve(filename) + val path = assumeReadableFile(fixturesDir, filename) val src = SourceCode.fromPath(path) val expectedOrigin = path.normalize().absolutePathString() assertEquals(expectedOrigin, src.origin) - - val expectedSrcText = path.toFile().readText() - val actualSrcText = src.getCharStream().toString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(path.toFile().readText(), src.asString()) } @Test fun testFromPathWithExistingNonNormalizedPath() { val filename = "simple_main.p8" val path = Path(".", "test", "..", "test", "fixtures", filename) + val srcFile = assumeReadableFile(path).toFile() val src = SourceCode.fromPath(path) val expectedOrigin = path.normalize().absolutePathString() assertEquals(expectedOrigin, src.origin) - - val expectedSrcText = path.toFile().readText() - val actualSrcText = src.getCharStream().toString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) } @Test fun testFromResourcesWithExistingP8File_withoutLeadingSlash() { val pathString = "prog8lib/math.p8" + val srcFile = assumeReadableFile(resourcesDir, pathString).toFile() val src = SourceCode.fromResources(pathString) assertEquals("@embedded@/$pathString", src.origin) - - val expectedSrcText = resourcesDir.resolve(pathString).toFile().readText() - val actualSrcText = src.asString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) } @Test fun testFromResourcesWithExistingP8File_withLeadingSlash() { val pathString = "/prog8lib/math.p8" + val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() val src = SourceCode.fromResources(pathString) assertEquals("@embedded@$pathString", src.origin) - - val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() - val actualSrcText = src.asString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) } @Test fun testFromResourcesWithExistingAsmFile_withoutLeadingSlash() { val pathString = "prog8lib/math.asm" + val srcFile = assumeReadableFile(resourcesDir, pathString).toFile() val src = SourceCode.fromResources(pathString) assertEquals("@embedded@/$pathString", src.origin) - - val expectedSrcText = resourcesDir.resolve(pathString).toFile().readText() - val actualSrcText = src.asString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) assertTrue(src.isFromResources, ".isFromResources") } @Test fun testFromResourcesWithExistingAsmFile_withLeadingSlash() { val pathString = "/prog8lib/math.asm" + val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() val src = SourceCode.fromResources(pathString) assertEquals("@embedded@$pathString", src.origin) - - val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() - val actualSrcText = src.asString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) } @Test fun testFromResourcesWithNonNormalizedPath() { val pathString = "/prog8lib/../prog8lib/math.p8" + val srcFile = assumeReadableFile(resourcesDir, pathString.substring(1)).toFile() val src = SourceCode.fromResources(pathString) assertEquals("@embedded@/prog8lib/math.p8", src.origin) - - val expectedSrcText = resourcesDir.resolve(pathString.substring(1)).toFile().readText() - val actualSrcText = src.asString() - assertEquals(expectedSrcText, actualSrcText) + assertEquals(srcFile.readText(), src.asString()) assertTrue(src.isFromResources, ".isFromResources") } @@ -146,15 +123,15 @@ class TestSourceCode { @Test fun testFromResourcesWithNonExistingFile_withLeadingSlash() { val pathString = "/prog8lib/i_do_not_exist" - val resPath = resourcesDir.resolve(pathString.substring(1)) - assumeNotExists(resPath) + assumeNotExists(resourcesDir, pathString.substring(1)) + assertThrows { SourceCode.fromResources(pathString) } } @Test fun testFromResourcesWithNonExistingFile_withoutLeadingSlash() { val pathString = "prog8lib/i_do_not_exist" - val resPath = resourcesDir.resolve(pathString) - assumeNotExists(resPath) + assumeNotExists(resourcesDir, pathString) + assertThrows { SourceCode.fromResources(pathString) } } @@ -162,8 +139,8 @@ class TestSourceCode { @Disabled("TODO: inside resources: cannot tell apart a folder from a file") fun testFromResourcesWithDirectory() { val pathString = "/prog8lib" + assumeDirectory(resourcesDir, pathString.substring(1)) assertThrows { SourceCode.fromResources(pathString) } } - } diff --git a/compilerAst/test/helpers/paths.kt b/compilerAst/test/helpers/paths.kt index 65c5de9fd..0e005cde5 100644 --- a/compilerAst/test/helpers/paths.kt +++ b/compilerAst/test/helpers/paths.kt @@ -6,33 +6,46 @@ import kotlin.io.path.* import java.nio.file.Path -val workingDir = Path("").absolute() // Note: Path(".") does NOT work..! -val fixturesDir = workingDir.resolve("test/fixtures") -val resourcesDir = workingDir.resolve("res") -val outputDir = workingDir.resolve("build/tmp/test") +val workingDir = assumeDirectory("").absolute() // Note: "." does NOT work..! +val fixturesDir = assumeDirectory(workingDir,"test/fixtures") +val resourcesDir = assumeDirectory(workingDir,"res") +val outputDir = assumeDirectory(workingDir, "build/tmp/test") -fun assumeReadable(path: Path) { - assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") -} - -fun assumeReadableFile(path: Path) { - assumeReadable(path) - assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") -} - -fun assumeDirectory(path: Path) { - assertTrue(path.isDirectory(), "sanity check; should be directory: $path") -} - -fun assumeNotExists(path: Path) { +fun assumeNotExists(path: Path): Path { assertFalse(path.exists(), "sanity check: should not exist: ${path.absolute()}") + return path } -fun sanityCheckDirectories(workingDirName: String? = null) { - if (workingDirName != null) - assertEquals(workingDirName, workingDir.fileName.toString(), "name of current working dir") - assumeDirectory(workingDir) - assumeDirectory(fixturesDir) - assumeDirectory(resourcesDir) - assumeDirectory(outputDir) +fun assumeNotExists(pathStr: String): Path = assumeNotExists(Path(pathStr)) +fun assumeNotExists(path: Path, other: String): Path = assumeNotExists(path.div(other)) + +fun assumeReadable(path: Path): Path { + assertTrue(path.isReadable(), "sanity check: should be readable: ${path.absolute()}") + return path } + +fun assumeReadableFile(path: Path): Path { + assertTrue(path.isRegularFile(), "sanity check: should be normal file: ${path.absolute()}") + return assumeReadable(path) +} + +fun assumeReadableFile(pathStr: String): Path = assumeReadableFile(Path(pathStr)) +fun assumeReadableFile(pathStr: String, other: Path): Path = assumeReadableFile(Path(pathStr), other) +fun assumeReadableFile(pathStr: String, other: String): Path = assumeReadableFile(Path(pathStr), other) +fun assumeReadableFile(path: Path, other: String): Path = assumeReadableFile(path.div(other)) +fun assumeReadableFile(path: Path, other: Path): Path = assumeReadableFile(path.div(other)) + +fun assumeDirectory(path: Path): Path { + assertTrue(path.isDirectory(), "sanity check; should be directory: $path") + return path +} + +fun assumeDirectory(pathStr: String): Path = assumeDirectory(Path(pathStr)) +fun assumeDirectory(path: Path, other: String): Path = assumeDirectory(path.div(other)) + + +@Deprecated("Directories are checked automatically at init.", + ReplaceWith("/* nothing */")) +@Suppress("unused") +fun sanityCheckDirectories(workingDirName: String? = null) { +} \ No newline at end of file diff --git a/compilerAst/test/helpers_pathsTests.kt b/compilerAst/test/helpers_pathsTests.kt new file mode 100644 index 000000000..418946b97 --- /dev/null +++ b/compilerAst/test/helpers_pathsTests.kt @@ -0,0 +1,311 @@ +package prog8tests + +import prog8tests.helpers.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.* + +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.assertThrows + +import kotlin.io.path.* +import kotlin.test.assertContains + +// Do not move into folder helpers/! +// This folder is also used by compiler/test +// but the testing of the helpers themselves must be performed ONLY HERE. +// +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class PathsHelpersTests { + + @Nested + inner class AssumeNotExists { + + @Nested + inner class WithOnePathArg { + + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThat("should return the path", + assumeNotExists(path), `is`(path)) + } + + @Test + fun testOnExistingFile() { + assertThrows { + assumeNotExists(fixturesDir.div("simple_main.p8")) + } + } + + @Test + fun testOnExistingDirectory() { + assertThrows { + assumeNotExists(fixturesDir) + } + } + } + + @Nested + inner class WithOneStringArg { + + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThat("should return the path", + assumeNotExists("$path"), `is`(path)) + } + + @Test + fun testOnExistingFile() { + val path = fixturesDir.div("simple_main.p8") + assertThrows { + assumeNotExists("$path") + } + } + + @Test + fun testOnExistingDirectory() { + assertThrows { + assumeNotExists("$fixturesDir") + } + } + } + + @Nested + inner class WithPathAndStringArgs { + + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThat("should return the path", + assumeNotExists(fixturesDir, "i_do_not_exist"), `is`(path)) + } + + @Test + fun testOnExistingFile() { + assertThrows { + assumeNotExists(fixturesDir, "simple_main.p8") + } + } + + @Test + fun testOnExistingDirectory() { + assertThrows { + assumeNotExists(fixturesDir, "..") + } + } + } + } + + @Nested + inner class AssumeDirectory { + + @Nested + inner class WithOnePathArg { + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThrows { + assumeDirectory(path) + } + } + + @Test + fun testOnExistingFile() { + val path = fixturesDir.div("simple_main.p8") + assertThrows { + assumeDirectory(path) + } + } + + @Test + fun testOnExistingDirectory() { + val path = workingDir + assertThat("should return the path", assumeDirectory(path), `is`(path)) + } + } + + @Nested + inner class WithOneStringArg { + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThrows { + assumeDirectory("$path") + } + } + + @Test + fun testOnExistingFile() { + val path = fixturesDir.div("simple_main.p8") + assertThrows { + assumeDirectory("$path") + } + } + + @Test + fun testOnExistingDirectory() { + val path = workingDir + assertThat("should return the path", + assumeDirectory("$path"), `is`(path)) + } + } + + @Nested + inner class WithPathAndStringArgs { + @Test + fun testOnNonExistingPath() { + assertThrows { + assumeDirectory(fixturesDir, "i_do_not_exist") + } + } + + @Test + fun testOnExistingFile() { + assertThrows { + assumeDirectory(fixturesDir, "simple_main.p8") + } + } + + @Test + fun testOnExistingDirectory() { + val path = workingDir.div("..") + assertThat( + "should return resulting path", + assumeDirectory(workingDir, ".."), `is`(path) + ) + } + } + } + + @Nested + inner class AssumeReadableFile { + + @Nested + inner class WithOnePathArg { + + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThrows { + assumeReadableFile(path) + } + } + + @Test + fun testOnReadableFile() { + val path = fixturesDir.div("simple_main.p8") + assertThat("should return the path", + assumeReadableFile(path), `is`(path)) + } + + @Test + fun testOnDirectory() { + assertThrows { + assumeReadableFile(fixturesDir) + } + } + } + + @Nested + inner class WithOneStringArg { + + @Test + fun testOnNonExistingPath() { + val path = fixturesDir.div("i_do_not_exist") + assertThrows { + assumeReadableFile("$path") + } + } + + @Test + fun testOnReadableFile() { + val path = fixturesDir.div("simple_main.p8") + assertThat("should return the resulting path", + assumeReadableFile("$path"), `is`(path)) + } + + @Test + fun testOnDirectory() { + assertThrows { + assumeReadableFile("$fixturesDir") + } + } + } + + @Nested + inner class WithPathAndStringArgs { + @Test + fun testOnNonexistingPath() { + assertThrows { + assumeReadableFile(fixturesDir, "i_do_not_exist") + } + } + + @Test + fun testOnReadableFile() { + val path = fixturesDir.div("simple_main.p8") + assertThat("should return the resulting path", + assumeReadableFile(fixturesDir, "simple_main.p8"), `is`(path)) + } + + @Test + fun testOnDirectory() { + assertThrows { + assumeReadableFile(fixturesDir, "..") + } + } + } + + @Nested + inner class WithPathAndPathArgs { + @Test + fun testOnNonexistingPath() { + assertThrows { + assumeReadableFile(fixturesDir, Path("i_do_not_exist")) + } + } + + @Test fun testOnReadableFile() { + assertThat("should return the resulting path", + assumeReadableFile(fixturesDir, Path("simple_main.p8")), + `is`(fixturesDir.div("simple_main.p8")) + ) + } + + @Test + fun testOnDirectory() { + assertThrows { + assumeReadableFile(fixturesDir, Path("..")) + } + } + } + + @Nested + inner class WithStringAndStringArgs { + @Test + fun testOnNonexistingPath() { + assertThrows { + assumeReadableFile("$fixturesDir", "i_do_not_exist") + } + } + + @Test + fun testOnReadableFile() { + assertThat("should return the resulting path", + assumeReadableFile(fixturesDir.toString(), "simple_main.p8"), + `is`(fixturesDir.div("simple_main.p8")) + ) + } + + @Test + fun testOnDirectory() { + assertThrows { + assumeReadableFile("$fixturesDir", "..") + } + } + } + } +} From c914f7bbcfa49e2741ad1e8171f8a219991d3ca6 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 10:11:29 +0200 Subject: [PATCH 45/68] + TestCompilerOptionLibdirs.kt: libdirs option doesn't seem to work --- compiler/test/TestCompilerOnExamples.kt | 8 +- compiler/test/TestCompilerOptionLibdirs.kt | 94 ++++++++++++++++++++++ compiler/test/fixtures/simple_main.p8 | 4 + compilerAst/test/helpers/paths.kt | 2 +- 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 compiler/test/TestCompilerOptionLibdirs.kt create mode 100644 compiler/test/fixtures/simple_main.p8 diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index e4d9a2254..0caa4d08c 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -32,8 +32,12 @@ class TestCompilerOnExamples { if (platform == Cx16Target) { searchIn.add(0, assumeDirectory(examplesDir, "cx16")) } - val filepath = searchIn.map { it.resolve("$name.p8") }.first { it.exists() } - val displayName = "${examplesDir.relativize(filepath)}: ${platform.name}, optimize=$optimize" + val filepath = searchIn + .map { it.resolve("$name.p8") } + .map { it.normalize().absolute() } + .map { workingDir.relativize(it) } + .first { it.exists() } + val displayName = "${examplesDir.relativize(filepath.absolute())}: ${platform.name}, optimize=$optimize" return dynamicTest(displayName) { compileProgram( filepath, diff --git a/compiler/test/TestCompilerOptionLibdirs.kt b/compiler/test/TestCompilerOptionLibdirs.kt new file mode 100644 index 000000000..4e577a085 --- /dev/null +++ b/compiler/test/TestCompilerOptionLibdirs.kt @@ -0,0 +1,94 @@ +package prog8tests + +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.AfterAll +import prog8tests.helpers.* +import kotlin.io.path.* +import java.nio.file.Path + +import prog8.compiler.compileProgram +import prog8.compiler.target.* + +/** + * ATTENTION: this is just kludge! + * They are not really unit tests, but rather tests of the whole process, + * from source file loading all the way through to running 64tass. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestCompilerOptionLibdirs { + + private lateinit var tempFileInWorkingDir: Path + + @BeforeAll + fun setUp() { + tempFileInWorkingDir = createTempFile(directory = workingDir, prefix = "tmp_", suffix = ".p8") + .also { it.writeText(""" + main { + sub start() { + } + } + """)} + } + + @AfterAll + fun tearDown() { + tempFileInWorkingDir.deleteExisting() + } + + private fun compileFile(filePath: Path, libdirs: List) = + compileProgram( + filepath = filePath, + optimize = false, + writeAssembly = true, + slowCodegenWarnings = false, + compilationTarget = Cx16Target.name, + libdirs, + outputDir + ) + + @Test + fun testAbsoluteFilePathInWorkingDir() { + val filepath = assumeReadableFile(tempFileInWorkingDir.absolute()) + compileFile(filepath, listOf()) + .assertSuccess() + } + + @Test + fun testFilePathInWorkingDirRelativeToWorkingDir() { + val filepath = assumeReadableFile(workingDir.relativize(tempFileInWorkingDir.absolute())) + compileFile(filepath, listOf()) + .assertSuccess() + } + + @Test + fun testFilePathInWorkingDirRelativeTo1stInLibdirs() { + val filepath = assumeReadableFile(tempFileInWorkingDir) + compileFile(filepath.fileName, listOf(workingDir.toString())) + .assertSuccess() + } + + @Test + fun testAbsoluteFilePathOutsideWorkingDir() { + val filepath = assumeReadableFile(fixturesDir, "simple_main.p8") + compileFile(filepath.absolute(), listOf()) + .assertSuccess() + } + + @Test + fun testFilePathOutsideWorkingDirRelativeToWorkingDir() { + val filepath = workingDir.relativize(assumeReadableFile(fixturesDir, "simple_main.p8").absolute()) + compileFile(filepath, listOf()) + .assertSuccess() + } + + @Test + fun testFilePathOutsideWorkingDirRelativeTo1stInLibdirs() { + val filepath = assumeReadableFile(fixturesDir, "simple_main.p8") + val libdirs = listOf("$fixturesDir") + compileFile(filepath.fileName, libdirs) + .assertSuccess() + } + +} diff --git a/compiler/test/fixtures/simple_main.p8 b/compiler/test/fixtures/simple_main.p8 new file mode 100644 index 000000000..afaa79f93 --- /dev/null +++ b/compiler/test/fixtures/simple_main.p8 @@ -0,0 +1,4 @@ +main { + sub start() { + } +} diff --git a/compilerAst/test/helpers/paths.kt b/compilerAst/test/helpers/paths.kt index 0e005cde5..85b29fb26 100644 --- a/compilerAst/test/helpers/paths.kt +++ b/compilerAst/test/helpers/paths.kt @@ -46,6 +46,6 @@ fun assumeDirectory(path: Path, other: String): Path = assumeDirectory(path.div( @Deprecated("Directories are checked automatically at init.", ReplaceWith("/* nothing */")) -@Suppress("unused") +@Suppress("UNUSED_PARAMETER") fun sanityCheckDirectories(workingDirName: String? = null) { } \ No newline at end of file From 0d06e3ff22537a676208f7dd3d3f47cf12d210d2 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 18 Jul 2021 19:02:47 +0200 Subject: [PATCH 46/68] */+ refactor tests of ModuleImporter, add some tests related to libdirs issue --- compilerAst/test/TestModuleImporter.kt | 155 ++++++++++++++++--------- 1 file changed, 100 insertions(+), 55 deletions(-) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 0466e8123..e3c1105ba 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -1,20 +1,68 @@ package prog8tests -import org.junit.jupiter.api.TestInstance -import org.junit.jupiter.api.Test -import kotlin.test.* import prog8tests.helpers.* - +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.* +import org.junit.jupiter.api.assertThrows import kotlin.io.path.* import prog8.ast.Program -import prog8.parser.ModuleImporter import prog8.parser.ParseError +import prog8.parser.ModuleImporter + @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestModuleImporter { + private val count = listOf("1st", "2nd", "3rd", "4th", "5th") + + @Test + fun testImportModuleWithExistingPath_absolute() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = assumeReadableFile(searchIn[0], fileName) + + val module = importer.importModule(path.absolute()) + assertThat(module.program, `is`(program)) + } + + @Test + fun testImportModuleWithExistingPath_relativeToWorkingDir() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = assumeReadableFile(searchIn[0], fileName) + assertThat("sanity check: path should NOT be absolute", path.isAbsolute, `is`(false)) + + val module = importer.importModule(path) + assertThat(module.program, `is`(program)) + } + + @Test + fun testImportModuleWithExistingPath_relativeTo1stDirInSearchList() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = Path(".", fileName) + assumeReadableFile(searchIn[0], path) + + val module = importer.importModule(path) + assertThat(module.program, `is`(program)) + } + @Test fun testImportModuleWithNonExistingPath() { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) @@ -22,7 +70,7 @@ class TestModuleImporter { val importer = ModuleImporter(program, "blah", listOf(searchIn)) val srcPath = assumeNotExists(fixturesDir, "i_do_not_exist") - assertFailsWith { importer.importModule(srcPath) } + assertThrows { importer.importModule(srcPath) } } @Test @@ -30,13 +78,13 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = assumeDirectory(fixturesDir) - // fn importModule(Path) used to check *.isReadable()*, but NOT .isRegularFile(): - assumeReadable(srcPath) - - assertFailsWith { importer.importModule(srcPath) } + assertThrows { importer.importModule(srcPath) } + .let { + assertThat(it.message!!, containsString("$srcPath")) + assertThat(it.file, `is`(srcPath.toFile())) + } } @Test @@ -44,19 +92,17 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") - val filename = "file_with_syntax_error.p8" - val path = assumeReadableFile(fixturesDir, filename) - val act = { importer.importModule(path) } + val act = { importer.importModule(srcPath) } - assertFailsWith { act() } - try { - act() - } catch (e: ParseError) { - assertEquals(path.absolutePathString(), e.position.file) - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + repeat (2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } } } @@ -65,21 +111,18 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") val act = { importer.importModule(importing) } - assertFailsWith { act() } - try { - act() - } catch (e: ParseError) { - val expectedProvenance = imported.absolutePathString() - assertEquals(expectedProvenance, e.position.file) - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based" ) - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + repeat (2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(imported.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } } } @@ -91,8 +134,16 @@ class TestModuleImporter { val filenameNoExt = assumeNotExists(fixturesDir, "i_do_not_exist").name val filenameWithExt = assumeNotExists(fixturesDir, "i_do_not_exist.p8").name - assertFailsWith { importer.importLibraryModule(filenameNoExt) } - assertFailsWith { importer.importLibraryModule(filenameWithExt) } + repeat (2) { n -> + assertThrows(count[n] + " call / NO .p8 extension") + { importer.importLibraryModule(filenameNoExt) }.let { + assertThat(it.message!!, containsString(filenameWithExt)) + } + assertThrows(count[n] + " call / with .p8 extension") + { importer.importLibraryModule(filenameWithExt) }.let { + assertThat(it.message!!, containsString(filenameWithExt)) + } + } } @Test @@ -102,17 +153,14 @@ class TestModuleImporter { val importer = ModuleImporter(program, "blah", listOf(searchIn)) val srcPath = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") - val act = { importer.importLibraryModule(srcPath.nameWithoutExtension) } - - assertFailsWith { act() } - try { - act() - } catch (e: ParseError) { - val expectedProvenance = srcPath.absolutePathString() - assertEquals(expectedProvenance, e.position.file) - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based") - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + repeat (2) { n -> + assertThrows (count[n] + " call") + { importer.importLibraryModule(srcPath.nameWithoutExtension) } .let { + assertThat(it.position.file, `is`(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } } } @@ -121,21 +169,18 @@ class TestModuleImporter { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") val imported = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") val act = { importer.importLibraryModule(importing.nameWithoutExtension) } - assertFailsWith { act() } - try { - act() - } catch (e: ParseError) { - val expectedProvenance = imported.normalize().absolutePathString() - assertEquals(expectedProvenance, e.position.file) - assertEquals(2, e.position.line, "line; should be 1-based") - assertEquals(6, e.position.startCol, "startCol; should be 0-based") - assertEquals(6, e.position.endCol, "endCol; should be 0-based") + repeat(2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(imported.normalize().absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } } } From 3b97a17648df01eb00ff284e76ad040e60fd7669 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 18 Jul 2021 19:03:51 +0200 Subject: [PATCH 47/68] * *little bit* of cleanup in ModuleImporter - *only refactoring* --- compiler/src/prog8/compiler/Compiler.kt | 5 ++--- compilerAst/src/prog8/parser/ModuleParsing.kt | 19 ++++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index 2fb3f03eb..3ddbd02f8 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -18,11 +18,10 @@ import prog8.compiler.target.asmGeneratorFor import prog8.optimizer.* import prog8.parser.ModuleImporter import prog8.parser.ParsingFailedError -import prog8.parser.moduleName import java.io.File import java.io.InputStream import java.nio.file.Path -import kotlin.io.path.Path +import kotlin.io.path.* import kotlin.system.measureTimeMillis @@ -176,7 +175,7 @@ private fun parseImports(filepath: Path, libdirs: List): Triple> { println("Compiler target: ${compTarget.name}. Parsing...") val bf = BuiltinFunctionsFacade(BuiltinFunctions) - val programAst = Program(moduleName(filepath.fileName), mutableListOf(), bf, compTarget) + val programAst = Program(filepath.nameWithoutExtension, mutableListOf(), bf, compTarget) bf.program = programAst val importer = ModuleImporter(programAst, compTarget.name, libdirs) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 78ac4f003..40a1a81f4 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -7,12 +7,9 @@ import prog8.ast.base.SyntaxError import prog8.ast.statements.Directive import prog8.ast.statements.DirectiveArg import java.io.File +import java.nio.file.Path import kotlin.io.FileSystemException -import java.nio.file.Path // TODO: use kotlin.io.paths.Path instead -import java.nio.file.Paths // TODO: use kotlin.io.paths.Path instead - - -fun moduleName(fileName: Path) = fileName.toString().substringBeforeLast('.') +import kotlin.io.path.* class ModuleImporter(private val program: Program, @@ -20,10 +17,10 @@ class ModuleImporter(private val program: Program, private val libdirs: List) { fun importModule(filePath: Path): Module { - print("importing '${moduleName(filePath.fileName)}'") + print("importing '${filePath.nameWithoutExtension}'") if (filePath.parent != null) { // TODO: use Path.relativize var importloc = filePath.toString() - val curdir = Paths.get("").toAbsolutePath().toString() + val curdir = Path("").absolutePathString() if(importloc.startsWith(curdir)) importloc = "." + importloc.substring(curdir.length) println(" (from '$importloc')") @@ -124,22 +121,22 @@ class ModuleImporter(private val program: Program, private fun tryGetModuleFromFile(name: String, importingModule: Module?): SourceCode? { val fileName = "$name.p8" - val libpaths = libdirs.map { Path.of(it) } + val libpaths = libdirs.map { Path(it) } val locations = if (importingModule == null) { // <=> imported from library module libpaths } else { libpaths.drop(1) + // TODO: why drop the first? // FIXME: won't work until Prog8Parser is fixed s.t. it fully initialzes the modules it returns - listOf(Path.of(importingModule.position.file).parent ?: Path.of(".")) + - listOf(Path.of(".", "prog8lib")) + listOf(Path(importingModule.position.file).parent ?: Path("")) + + listOf(Path(".", "prog8lib")) } locations.forEach { try { return SourceCode.fromPath(it.resolve(fileName)) } catch (e: NoSuchFileException) { - } + } } //throw ParsingFailedError("$position Import: no module source file '$fileName' found (I've looked in: embedded libs and $locations)") From b8fade23de7a1b987910f59456b27743d905414a Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 15:15:54 +0200 Subject: [PATCH 48/68] * (first quick) fix: ModuleImporter should look in given "libdirs" (or better "srcdirs"?) for module file --- compilerAst/src/prog8/parser/ModuleParsing.kt | 35 +- compilerAst/src/prog8/parser/SourceCode.kt | 13 +- compilerAst/test/TestModuleImporter.kt | 363 +++++++++++------- compilerAst/test/helpers/paths.kt | 2 + compilerAst/test/helpers_pathsTests.kt | 52 +++ 5 files changed, 312 insertions(+), 153 deletions(-) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index 40a1a81f4..c66d95e10 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -14,21 +14,31 @@ import kotlin.io.path.* class ModuleImporter(private val program: Program, private val compilationTargetName: String, - private val libdirs: List) { + libdirs: List) { + + private val libpaths: List = libdirs.map { Path(it) } fun importModule(filePath: Path): Module { - print("importing '${filePath.nameWithoutExtension}'") - if (filePath.parent != null) { // TODO: use Path.relativize - var importloc = filePath.toString() - val curdir = Path("").absolutePathString() - if(importloc.startsWith(curdir)) - importloc = "." + importloc.substring(curdir.length) - println(" (from '$importloc')") - } - else - println("") + val currentDir = Path("").absolute() + val searchIn = listOf(currentDir) + libpaths + val candidates = searchIn + .map { it.absolute().div(filePath).normalize().absolute() } + .filter { it.exists() } + .map { currentDir.relativize(it) } + .map { if (it.isAbsolute) it else Path(".", "$it") } - val module = Prog8Parser.parseModule(SourceCode.fromPath(filePath)) + val srcPath = when (candidates.size) { + 0 -> throw NoSuchFileException( + file = filePath.normalize().toFile(), + reason = "searched in $searchIn") + 1 -> candidates.first() + else -> candidates.first() // TODO: report error if more than 1 candidate? + } + + var logMsg = "importing '${filePath.nameWithoutExtension}' (from $srcPath)" + println(logMsg) + + val module = Prog8Parser.parseModule(SourceCode.fromPath(srcPath)) module.program = program module.linkParents(program.namespace) @@ -121,7 +131,6 @@ class ModuleImporter(private val program: Program, private fun tryGetModuleFromFile(name: String, importingModule: Module?): SourceCode? { val fileName = "$name.p8" - val libpaths = libdirs.map { Path(it) } val locations = if (importingModule == null) { // <=> imported from library module libpaths diff --git a/compilerAst/src/prog8/parser/SourceCode.kt b/compilerAst/src/prog8/parser/SourceCode.kt index 44f4fdb25..57d1a7724 100644 --- a/compilerAst/src/prog8/parser/SourceCode.kt +++ b/compilerAst/src/prog8/parser/SourceCode.kt @@ -85,13 +85,14 @@ abstract class SourceCode { * @throws AccessDeniedException if the given path points to a directory or the file is non-readable for some other reason */ fun fromPath(path: Path): SourceCode { - if (!path.exists()) - throw NoSuchFileException(path.toFile()) - if (path.isDirectory()) - throw AccessDeniedException(path.toFile(), reason = "Not a file but a directory") - if (!path.isReadable()) - throw AccessDeniedException(path.toFile(), reason = "Is not readable") val normalized = path.normalize() + val file = normalized.toFile() + if (!path.exists()) + throw NoSuchFileException(file) + if (path.isDirectory()) + throw AccessDeniedException(file, reason = "Not a file but a directory") + if (!path.isReadable()) + throw AccessDeniedException(file, reason = "Is not readable") return object : SourceCode() { override val isFromResources = false override val origin = normalized.absolutePathString() diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index e3c1105ba..16edb94c4 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -5,6 +5,8 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.* +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Nested import org.junit.jupiter.api.assertThrows import kotlin.io.path.* @@ -16,172 +18,265 @@ import prog8.parser.ModuleImporter @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestModuleImporter { - private val count = listOf("1st", "2nd", "3rd", "4th", "5th") - @Test - fun testImportModuleWithExistingPath_absolute() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = listOf( - Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front - ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) - val fileName = "simple_main.p8" - val path = assumeReadableFile(searchIn[0], fileName) + @Nested + inner class Constructor { - val module = importer.importModule(path.absolute()) - assertThat(module.program, `is`(program)) + @Test + @Disabled("TODO: invalid entries in search list") + fun testInvalidEntriesInSearchList() {} + + @Test + @Disabled("TODO: literal duplicates in search list") + fun testLiteralDuplicatesInSearchList() {} + + @Test + @Disabled("TODO: factual duplicates in search list") + fun testFactualDuplicatesInSearchList() {} } - @Test - fun testImportModuleWithExistingPath_relativeToWorkingDir() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = listOf( - Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front - ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) - val fileName = "simple_main.p8" - val path = assumeReadableFile(searchIn[0], fileName) - assertThat("sanity check: path should NOT be absolute", path.isAbsolute, `is`(false)) + @Nested + inner class ImportModule { - val module = importer.importModule(path) - assertThat(module.program, `is`(program)) - } + @Nested + inner class WithInvalidPath { + @Test + fun testNonexisting() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val dirRel = assumeDirectory(".", workingDir.relativize(fixturesDir)) + val searchIn = dirRel.invariantSeparatorsPathString + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val srcPathRel = assumeNotExists(dirRel, "i_do_not_exist") + val srcPathAbs = srcPathRel.absolute() - @Test - fun testImportModuleWithExistingPath_relativeTo1stDirInSearchList() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = listOf( - Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front - ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) - val fileName = "simple_main.p8" - val path = Path(".", fileName) - assumeReadableFile(searchIn[0], path) + assertThrows { importer.importModule(srcPathRel) } + .let { + assertThat( + ".file should be normalized", + "${it.file}", `is`("${it.file.normalize()}") + ) + assertThat( + ".file should point to specified path", + it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + ) + } - val module = importer.importModule(path) - assertThat(module.program, `is`(program)) - } - - @Test - fun testImportModuleWithNonExistingPath() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = assumeNotExists(fixturesDir, "i_do_not_exist") - - assertThrows { importer.importModule(srcPath) } - } - - @Test - fun testImportModuleWithDirectoryPath() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = assumeDirectory(fixturesDir) - - assertThrows { importer.importModule(srcPath) } - .let { - assertThat(it.message!!, containsString("$srcPath")) - assertThat(it.file, `is`(srcPath.toFile())) + assertThrows { importer.importModule(srcPathAbs) } + .let { + assertThat( + ".file should be normalized", + "${it.file}", `is`("${it.file.normalize()}") + ) + assertThat( + ".file should point to specified path", + it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + ) + } } - } - @Test - fun testImportModuleWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + @Test + fun testDirectory() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val dirRel = assumeDirectory(workingDir.relativize(fixturesDir)) + val searchIn = Path(".", "$dirRel").invariantSeparatorsPathString + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val srcPathRel = dirRel + val srcPathAbs = srcPathRel.absolute() - val act = { importer.importModule(srcPath) } + assertThrows { importer.importModule(srcPathRel) } + .let { + assertThat( + ".file should be normalized", + "${it.file}", `is`("${it.file.normalize()}") + ) + assertThat( + ".file should point to specified path", + it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + ) + } - repeat (2) { n -> - assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(srcPath.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + assertThrows { importer.importModule(srcPathAbs) } + .let { + assertThat( + ".file should be normalized", + "${it.file}", `is`("${it.file.normalize()}") + ) + assertThat( + ".file should point to specified path", + it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + ) + } } } - } - @Test - fun testImportModuleWithImportingModuleWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") - val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + @Nested + inner class WithValidPath { - val act = { importer.importModule(importing) } + @Test + fun testAbsolute() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = assumeReadableFile(searchIn[0], fileName) - repeat (2) { n -> - assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(imported.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + val module = importer.importModule(path.absolute()) + assertThat(module.program, `is`(program)) } - } - } - @Test - fun testImportLibraryModuleWithNonExistingName() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val filenameNoExt = assumeNotExists(fixturesDir, "i_do_not_exist").name - val filenameWithExt = assumeNotExists(fixturesDir, "i_do_not_exist.p8").name + @Test + fun testRelativeToWorkingDir() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = assumeReadableFile(searchIn[0], fileName) + assertThat("sanity check: path should NOT be absolute", path.isAbsolute, `is`(false)) - repeat (2) { n -> - assertThrows(count[n] + " call / NO .p8 extension") - { importer.importLibraryModule(filenameNoExt) }.let { - assertThat(it.message!!, containsString(filenameWithExt)) + val module = importer.importModule(path) + assertThat(module.program, `is`(program)) + } + + @Test + fun testRelativeTo1stDirInSearchList() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = listOf( + Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front + ).map { it.invariantSeparatorsPathString } + val importer = ModuleImporter(program, "blah", searchIn) + val fileName = "simple_main.p8" + val path = Path(".", fileName) + assumeReadableFile(searchIn[0], path) + + val module = importer.importModule(path) + assertThat(module.program, `is`(program)) + } + + @Test + @Disabled("TODO: relative to 2nd in search list") + fun testRelativeTo2ndDirInSearchList() {} + + @Test + @Disabled("TODO: ambiguous - 2 or more really different candidates") + fun testAmbiguousCandidates() {} + + @Nested + inner class WithBadFile { + @Test + fun testWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + + val act = { importer.importModule(srcPath) } + + repeat(2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } + } } - assertThrows(count[n] + " call / with .p8 extension") - { importer.importLibraryModule(filenameWithExt) }.let { - assertThat(it.message!!, containsString(filenameWithExt)) + + @Test + fun testImportingFileWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") + val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + + val act = { importer.importModule(importing) } + + repeat(2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(imported.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } + } } + } } + } - @Test - fun testImportLibraryModuleWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val srcPath = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") + @Nested + inner class ImportLibraryModule { + @Nested + inner class WithInvalidName { + @Test + fun testWithNonExistingName() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val filenameNoExt = assumeNotExists(fixturesDir, "i_do_not_exist").name + val filenameWithExt = assumeNotExists(fixturesDir, "i_do_not_exist.p8").name - repeat (2) { n -> - assertThrows (count[n] + " call") - { importer.importLibraryModule(srcPath.nameWithoutExtension) } .let { - assertThat(it.position.file, `is`(srcPath.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + repeat(2) { n -> + assertThrows(count[n] + " call / NO .p8 extension") + { importer.importLibraryModule(filenameNoExt) }.let { + assertThat(it.message!!, containsString(filenameWithExt)) + } + assertThrows(count[n] + " call / with .p8 extension") + { importer.importLibraryModule(filenameWithExt) }.let { + assertThat(it.message!!, containsString(filenameWithExt)) + } } + } } - } - @Test - fun testImportLibraryModuleWithImportingBadModule() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) - val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") - val imported = assumeReadableFile(fixturesDir,"file_with_syntax_error.p8") + @Nested + inner class WithValidName { + @Nested + inner class WithBadFile { + @Test + fun testWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") - val act = { importer.importLibraryModule(importing.nameWithoutExtension) } + repeat(2) { n -> + assertThrows(count[n] + " call") + { importer.importLibraryModule(srcPath.nameWithoutExtension) }.let { + assertThat(it.position.file, `is`(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } + } + } - repeat(2) { n -> - assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(imported.normalize().absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + @Test + fun testImportingFileWithSyntaxError() { + val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") + val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") + val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + + val act = { importer.importLibraryModule(importing.nameWithoutExtension) } + + repeat(2) { n -> + assertThrows(count[n] + " call") { act() }.let { + assertThat(it.position.file, `is`(imported.normalize().absolutePathString())) + assertThat("line; should be 1-based", it.position.line, `is`(2)) + assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) + assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + } + } + } } } } - } diff --git a/compilerAst/test/helpers/paths.kt b/compilerAst/test/helpers/paths.kt index 85b29fb26..4e99518b0 100644 --- a/compilerAst/test/helpers/paths.kt +++ b/compilerAst/test/helpers/paths.kt @@ -42,6 +42,8 @@ fun assumeDirectory(path: Path): Path { fun assumeDirectory(pathStr: String): Path = assumeDirectory(Path(pathStr)) fun assumeDirectory(path: Path, other: String): Path = assumeDirectory(path.div(other)) +fun assumeDirectory(pathStr: String, other: String): Path = assumeDirectory(Path(pathStr).div(other)) +fun assumeDirectory(pathStr: String, other: Path): Path = assumeDirectory(Path(pathStr).div(other)) @Deprecated("Directories are checked automatically at init.", diff --git a/compilerAst/test/helpers_pathsTests.kt b/compilerAst/test/helpers_pathsTests.kt index 418946b97..f85fb8363 100644 --- a/compilerAst/test/helpers_pathsTests.kt +++ b/compilerAst/test/helpers_pathsTests.kt @@ -178,6 +178,58 @@ class PathsHelpersTests { ) } } + + @Nested + inner class WithStringAndStringArgs { + @Test + fun testOnNonExistingPath() { + assertThrows { + assumeDirectory("$fixturesDir", "i_do_not_exist") + } + } + + @Test + fun testOnExistingFile() { + assertThrows { + assumeDirectory("$fixturesDir", "simple_main.p8") + } + } + + @Test + fun testOnExistingDirectory() { + val path = workingDir.div("..") + assertThat( + "should return resulting path", + assumeDirectory("$workingDir", ".."), `is`(path) + ) + } + } + + @Nested + inner class WithStringAndPathArgs { + @Test + fun testOnNonExistingPath() { + assertThrows { + assumeDirectory("$fixturesDir", Path("i_do_not_exist")) + } + } + + @Test + fun testOnExistingFile() { + assertThrows { + assumeDirectory("$fixturesDir", Path("simple_main.p8")) + } + } + + @Test + fun testOnExistingDirectory() { + val path = workingDir.div("..") + assertThat( + "should return resulting path", + assumeDirectory("$workingDir", Path("..")), `is`(path) + ) + } + } } @Nested From f2cb89a128054c3a0c2816e2d3860b11619ff240 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 15:37:57 +0200 Subject: [PATCH 49/68] - ModuleImporter: deduplicate code --- compilerAst/src/prog8/parser/ModuleParsing.kt | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleParsing.kt index c66d95e10..32a2e9646 100644 --- a/compilerAst/src/prog8/parser/ModuleParsing.kt +++ b/compilerAst/src/prog8/parser/ModuleParsing.kt @@ -38,21 +38,7 @@ class ModuleImporter(private val program: Program, var logMsg = "importing '${filePath.nameWithoutExtension}' (from $srcPath)" println(logMsg) - val module = Prog8Parser.parseModule(SourceCode.fromPath(srcPath)) - - module.program = program - module.linkParents(program.namespace) - program.modules.add(module) - - // accept additional imports - val lines = module.statements.toMutableList() - lines.asSequence() - .mapIndexed { i, it -> i to it } - .filter { (it.second as? Directive)?.directive == "%import" } - .forEach { executeImportDirective(it.second as Directive, module) } - - module.statements = lines - return module + return importModule(SourceCode.fromPath(srcPath)) } fun importLibraryModule(name: String): Module? { @@ -88,7 +74,7 @@ class ModuleImporter(private val program: Program, throw SyntaxError("cannot import self", import.position) val existing = program.modules.singleOrNull { it.name == moduleName } - if(existing!=null) + if (existing!=null) return null // TODO: why return null instead of Module instance? var srcCode = tryGetModuleFromResource("$moduleName.p8", compilationTargetName) From d7dd7f70c0e21775c62ec22003920da9a09e6a25 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 18 Jul 2021 19:18:30 +0200 Subject: [PATCH 50/68] * rename file ModuleParsing.kt to ModuleImporter.kt (nothing else, still in compilerAst) --- .../src/prog8/parser/{ModuleParsing.kt => ModuleImporter.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename compilerAst/src/prog8/parser/{ModuleParsing.kt => ModuleImporter.kt} (100%) diff --git a/compilerAst/src/prog8/parser/ModuleParsing.kt b/compilerAst/src/prog8/parser/ModuleImporter.kt similarity index 100% rename from compilerAst/src/prog8/parser/ModuleParsing.kt rename to compilerAst/src/prog8/parser/ModuleImporter.kt From ebe04fc11421074295d645ebda3f6de9d4e633d0 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 16:26:27 +0200 Subject: [PATCH 51/68] * @Disable ModuleImporter test re importing a faulty module twice - no easy fix for this atm --- compilerAst/test/TestModuleImporter.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 16edb94c4..61a0db0b1 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -257,8 +257,8 @@ class TestModuleImporter { } } - @Test - fun testImportingFileWithSyntaxError() { + + private fun doTestImportingFileWithSyntaxError(repetitions: Int) { val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") val importer = ModuleImporter(program, "blah", listOf(searchIn)) @@ -267,7 +267,7 @@ class TestModuleImporter { val act = { importer.importLibraryModule(importing.nameWithoutExtension) } - repeat(2) { n -> + repeat(repetitions) { n -> assertThrows(count[n] + " call") { act() }.let { assertThat(it.position.file, `is`(imported.normalize().absolutePathString())) assertThat("line; should be 1-based", it.position.line, `is`(2)) @@ -276,6 +276,17 @@ class TestModuleImporter { } } } + + @Test + fun testImportingFileWithSyntaxError_once() { + doTestImportingFileWithSyntaxError(1) + } + + @Test + @Disabled("TODO: module that imports faulty module should not be kept in Program.modules") + fun testImportingFileWithSyntaxError_twice() { + doTestImportingFileWithSyntaxError(2) + } } } } From 007d8d28110d42470e06412e003cadd227e64672 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 17:24:12 +0200 Subject: [PATCH 52/68] * ModuleImporter tests: refactor, more precise assertions about the program's modules --- compilerAst/test/TestModuleImporter.kt | 146 ++++++++++++++----------- 1 file changed, 84 insertions(+), 62 deletions(-) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index 61a0db0b1..c35b026ab 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -1,10 +1,12 @@ package prog8tests import prog8tests.helpers.* +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.equalTo +import org.hamcrest.Matchers.containsString import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance -import org.hamcrest.MatcherAssert.assertThat -import org.hamcrest.Matchers.* +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Nested import org.junit.jupiter.api.assertThrows @@ -14,12 +16,26 @@ import prog8.ast.Program import prog8.parser.ParseError import prog8.parser.ModuleImporter +import kotlin.test.assertContains @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestModuleImporter { private val count = listOf("1st", "2nd", "3rd", "4th", "5th") + lateinit var program: Program + @BeforeEach + fun beforeEach() { + program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + } + + private fun makeImporter(vararg searchIn: String): ModuleImporter = makeImporter(searchIn.asList()) + + private fun makeImporter(searchIn: Iterable) = ModuleImporter( + program, + "blah", + searchIn.toList()) + @Nested inner class Constructor { @@ -43,10 +59,8 @@ class TestModuleImporter { inner class WithInvalidPath { @Test fun testNonexisting() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val dirRel = assumeDirectory(".", workingDir.relativize(fixturesDir)) - val searchIn = dirRel.invariantSeparatorsPathString - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val importer = makeImporter(dirRel.invariantSeparatorsPathString) val srcPathRel = assumeNotExists(dirRel, "i_do_not_exist") val srcPathAbs = srcPathRel.absolute() @@ -54,33 +68,34 @@ class TestModuleImporter { .let { assertThat( ".file should be normalized", - "${it.file}", `is`("${it.file.normalize()}") + "${it.file}", equalTo("${it.file.normalize()}") ) assertThat( ".file should point to specified path", - it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + it.file.absolutePath, equalTo("${srcPathAbs.normalize()}") ) } + assertThat(program.modules.size, equalTo(1)) assertThrows { importer.importModule(srcPathAbs) } .let { assertThat( ".file should be normalized", - "${it.file}", `is`("${it.file.normalize()}") + "${it.file}", equalTo("${it.file.normalize()}") ) assertThat( ".file should point to specified path", - it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + it.file.absolutePath, equalTo("${srcPathAbs.normalize()}") ) } + assertThat(program.modules.size, equalTo(1)) } @Test fun testDirectory() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val dirRel = assumeDirectory(workingDir.relativize(fixturesDir)) val searchIn = Path(".", "$dirRel").invariantSeparatorsPathString - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val importer = makeImporter(searchIn) val srcPathRel = dirRel val srcPathAbs = srcPathRel.absolute() @@ -88,25 +103,27 @@ class TestModuleImporter { .let { assertThat( ".file should be normalized", - "${it.file}", `is`("${it.file.normalize()}") + "${it.file}", equalTo("${it.file.normalize()}") ) assertThat( ".file should point to specified path", - it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + it.file.absolutePath, equalTo("${srcPathAbs.normalize()}") ) } + assertThat(program.modules.size, equalTo(1)) assertThrows { importer.importModule(srcPathAbs) } .let { assertThat( ".file should be normalized", - "${it.file}", `is`("${it.file.normalize()}") + "${it.file}", equalTo("${it.file.normalize()}") ) assertThat( ".file should point to specified path", - it.file.absolutePath, `is`("${srcPathAbs.normalize()}") + it.file.absolutePath, equalTo("${srcPathAbs.normalize()}") ) } + assertThat(program.modules.size, equalTo(1)) } } @@ -115,46 +132,49 @@ class TestModuleImporter { @Test fun testAbsolute() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = listOf( Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) + val importer = makeImporter(searchIn) val fileName = "simple_main.p8" val path = assumeReadableFile(searchIn[0], fileName) val module = importer.importModule(path.absolute()) - assertThat(module.program, `is`(program)) + assertThat(program.modules.size, equalTo(2)) + assertContains(program.modules, module) + assertThat(module.program, equalTo(program)) } @Test fun testRelativeToWorkingDir() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) val searchIn = listOf( Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) + val importer = makeImporter(searchIn) val fileName = "simple_main.p8" val path = assumeReadableFile(searchIn[0], fileName) - assertThat("sanity check: path should NOT be absolute", path.isAbsolute, `is`(false)) + assertThat("sanity check: path should NOT be absolute", path.isAbsolute, equalTo(false)) val module = importer.importModule(path) - assertThat(module.program, `is`(program)) + assertThat(program.modules.size, equalTo(2)) + assertContains(program.modules, module) + assertThat(module.program, equalTo(program)) } @Test fun testRelativeTo1stDirInSearchList() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = listOf( - Path(".").div(workingDir.relativize(fixturesDir)), // we do want a dot "." in front - ).map { it.invariantSeparatorsPathString } - val importer = ModuleImporter(program, "blah", searchIn) + val searchIn = Path(".") + .div(workingDir.relativize(fixturesDir)) + .invariantSeparatorsPathString + val importer = makeImporter(searchIn) val fileName = "simple_main.p8" val path = Path(".", fileName) - assumeReadableFile(searchIn[0], path) + assumeReadableFile(searchIn, path) val module = importer.importModule(path) - assertThat(module.program, `is`(program)) + assertThat(program.modules.size, equalTo(2)) + assertContains(program.modules, module) + assertThat(module.program, equalTo(program)) } @Test @@ -169,28 +189,27 @@ class TestModuleImporter { inner class WithBadFile { @Test fun testWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) + val importer = makeImporter(searchIn.invariantSeparatorsPathString) val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") val act = { importer.importModule(srcPath) } repeat(2) { n -> assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(srcPath.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + assertThat(it.position.file, equalTo(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, equalTo(2)) + assertThat("startCol; should be 0-based", it.position.startCol, equalTo(6)) + assertThat("endCol; should be 0-based", it.position.endCol, equalTo(6)) } + assertThat(program.modules.size, equalTo(1)) } } @Test fun testImportingFileWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) + val importer = makeImporter(searchIn.invariantSeparatorsPathString) val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") @@ -198,11 +217,12 @@ class TestModuleImporter { repeat(2) { n -> assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(imported.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + assertThat(it.position.file, equalTo(imported.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, equalTo(2)) + assertThat("startCol; should be 0-based", it.position.startCol, equalTo(6)) + assertThat("endCol; should be 0-based", it.position.endCol, equalTo(6)) } +// TODO("assertThat(program.modules.size, equalTo(2))") } } } @@ -216,9 +236,8 @@ class TestModuleImporter { inner class WithInvalidName { @Test fun testWithNonExistingName() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) + val importer = makeImporter(searchIn.invariantSeparatorsPathString) val filenameNoExt = assumeNotExists(fixturesDir, "i_do_not_exist").name val filenameWithExt = assumeNotExists(fixturesDir, "i_do_not_exist.p8").name @@ -227,10 +246,13 @@ class TestModuleImporter { { importer.importLibraryModule(filenameNoExt) }.let { assertThat(it.message!!, containsString(filenameWithExt)) } + assertThat(program.modules.size, equalTo(1)) + assertThrows(count[n] + " call / with .p8 extension") { importer.importLibraryModule(filenameWithExt) }.let { assertThat(it.message!!, containsString(filenameWithExt)) } + assertThat(program.modules.size, equalTo(1)) } } } @@ -241,27 +263,26 @@ class TestModuleImporter { inner class WithBadFile { @Test fun testWithSyntaxError() { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) + val importer = makeImporter(searchIn.invariantSeparatorsPathString) val srcPath = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") repeat(2) { n -> assertThrows(count[n] + " call") - { importer.importLibraryModule(srcPath.nameWithoutExtension) }.let { - assertThat(it.position.file, `is`(srcPath.absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) - } + { importer.importLibraryModule(srcPath.nameWithoutExtension) }.let { + assertThat(it.position.file, equalTo(srcPath.absolutePathString())) + assertThat("line; should be 1-based", it.position.line, equalTo(2)) + assertThat("startCol; should be 0-based", it.position.startCol, equalTo(6)) + assertThat("endCol; should be 0-based", it.position.endCol, equalTo(6)) + } + assertThat(program.modules.size, equalTo(1)) } } private fun doTestImportingFileWithSyntaxError(repetitions: Int) { - val program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) - val searchIn = "./" + workingDir.relativize(fixturesDir).toString().replace("\\", "/") - val importer = ModuleImporter(program, "blah", listOf(searchIn)) + val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) + val importer = makeImporter(searchIn.invariantSeparatorsPathString) val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") val imported = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") @@ -269,11 +290,12 @@ class TestModuleImporter { repeat(repetitions) { n -> assertThrows(count[n] + " call") { act() }.let { - assertThat(it.position.file, `is`(imported.normalize().absolutePathString())) - assertThat("line; should be 1-based", it.position.line, `is`(2)) - assertThat("startCol; should be 0-based", it.position.startCol, `is`(6)) - assertThat("endCol; should be 0-based", it.position.endCol, `is`(6)) + assertThat(it.position.file, equalTo(imported.normalize().absolutePathString())) + assertThat("line; should be 1-based", it.position.line, equalTo(2)) + assertThat("startCol; should be 0-based", it.position.startCol, equalTo(6)) + assertThat("endCol; should be 0-based", it.position.endCol, equalTo(6)) } +// TODO("assertThat(program.modules.size, equalTo(1))") } } From eb46852bb941720a82a2ee90bc8d855e45c92a09 Mon Sep 17 00:00:00 2001 From: meisl Date: Sun, 1 Aug 2021 22:47:11 +0200 Subject: [PATCH 53/68] * restrict access to Program.modules, add tests --- compiler/src/prog8/compiler/Compiler.kt | 4 +- .../compiler/astprocessing/AstExtensions.kt | 4 +- compiler/test/AsmgenTests.kt | 6 +- compiler/test/TestMemory.kt | 38 +++--- compilerAst/src/prog8/ast/AstToplevel.kt | 82 +++++++----- .../src/prog8/parser/ModuleImporter.kt | 4 +- compilerAst/test/TestAstToSourceCode.kt | 5 +- compilerAst/test/TestModuleImporter.kt | 31 +++-- compilerAst/test/ast/ProgramTests.kt | 118 ++++++++++++++++++ 9 files changed, 218 insertions(+), 74 deletions(-) create mode 100644 compilerAst/test/ast/ProgramTests.kt diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index 3ddbd02f8..557b23a2a 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -135,7 +135,7 @@ fun compileProgram(filepath: Path, throw x } - val failedProgram = Program("failed", mutableListOf(), BuiltinFunctionsFacade(BuiltinFunctions), compTarget) + val failedProgram = Program("failed", BuiltinFunctionsFacade(BuiltinFunctions), compTarget) return CompilationResult(false, failedProgram, programName, compTarget, emptyList()) } @@ -175,7 +175,7 @@ private fun parseImports(filepath: Path, libdirs: List): Triple> { println("Compiler target: ${compTarget.name}. Parsing...") val bf = BuiltinFunctionsFacade(BuiltinFunctions) - val programAst = Program(filepath.nameWithoutExtension, mutableListOf(), bf, compTarget) + val programAst = Program(filepath.nameWithoutExtension, bf, compTarget) bf.program = programAst val importer = ModuleImporter(programAst, compTarget.name, libdirs) diff --git a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt index 8f9ed35fe..92b7c1285 100644 --- a/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt +++ b/compiler/src/prog8/compiler/astprocessing/AstExtensions.kt @@ -163,9 +163,7 @@ internal fun Program.moveMainAndStartToFirst() { val start = this.entrypoint() val mod = start.definingModule() val block = start.definingBlock() - if(!modules.remove(mod)) - throw FatalAstException("module wrong") - modules.add(0, mod) + moveModuleToFront(mod) mod.remove(block) var afterDirective = mod.statements.indexOfFirst { it !is Directive } if(afterDirective<0) diff --git a/compiler/test/AsmgenTests.kt b/compiler/test/AsmgenTests.kt index 34ecf58cb..4bbd85479 100644 --- a/compiler/test/AsmgenTests.kt +++ b/compiler/test/AsmgenTests.kt @@ -66,9 +66,9 @@ locallabel: val block = Block("main", null, mutableListOf(labelInBlock, varInBlock, subroutine), false, Position.DUMMY) val module = Module("test", mutableListOf(block), Position.DUMMY, null) - module.linkParents(ParentSentinel) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.program = program + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program.namespace)?! return program } diff --git a/compiler/test/TestMemory.kt b/compiler/test/TestMemory.kt index 25a1bf3bb..598c0bb5f 100644 --- a/compiler/test/TestMemory.kt +++ b/compiler/test/TestMemory.kt @@ -24,7 +24,7 @@ class TestMemory { var memexpr = NumericLiteralValue.optimalInteger(0x0000, Position.DUMMY) var target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) + val program = Program("test", DummyFunctions, DummyMemsizer) assertTrue(C64Target.isInRegularRAM(target, program)) memexpr = NumericLiteralValue.optimalInteger(0x1000, Position.DUMMY) @@ -49,7 +49,7 @@ class TestMemory { var memexpr = NumericLiteralValue.optimalInteger(0xa000, Position.DUMMY) var target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) + val program = Program("test", DummyFunctions, DummyMemsizer) assertFalse(C64Target.isInRegularRAM(target, program)) memexpr = NumericLiteralValue.optimalInteger(0xafff, Position.DUMMY) @@ -68,7 +68,7 @@ class TestMemory { @Test fun testInValidRamC64_memory_identifiers() { var target = createTestProgramForMemoryRefViaVar(0x1000, VarDeclType.VAR) - val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) + val program = Program("test", DummyFunctions, DummyMemsizer) assertTrue(C64Target.isInRegularRAM(target, program)) target = createTestProgramForMemoryRefViaVar(0xd020, VarDeclType.VAR) @@ -97,7 +97,7 @@ class TestMemory { fun testInValidRamC64_memory_expression() { val memexpr = PrefixExpression("+", NumericLiteralValue.optimalInteger(0x1000, Position.DUMMY), Position.DUMMY) val target = AssignTarget(null, null, DirectMemoryWrite(memexpr, Position.DUMMY), Position.DUMMY) - val program = Program("test", mutableListOf(), DummyFunctions, DummyMemsizer) + val program = Program("test", DummyFunctions, DummyMemsizer) assertFalse(C64Target.isInRegularRAM(target, program)) } @@ -108,8 +108,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -121,8 +122,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -134,8 +136,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertFalse(C64Target.isInRegularRAM(target, program)) } @@ -147,8 +150,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -161,8 +165,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertTrue(C64Target.isInRegularRAM(target, program)) } @@ -175,8 +180,9 @@ class TestMemory { val assignment = Assignment(target, NumericLiteralValue.optimalInteger(0, Position.DUMMY), Position.DUMMY) val subroutine = Subroutine("test", emptyList(), emptyList(), emptyList(), emptyList(), emptySet(), null, false, false, mutableListOf(decl, assignment), Position.DUMMY) val module = Module("test", mutableListOf(subroutine), Position.DUMMY, null) - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(ParentSentinel) + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) + module.linkParents(ParentSentinel) // TODO: why not module.linkParents(program) or .linkParents(program.namespace)? assertFalse(C64Target.isInRegularRAM(target, program)) } } diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 4c67f054c..861f80f68 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -230,40 +230,61 @@ interface Node { /*********** Everything starts from here, the Program; zero or more modules *************/ class Program(val name: String, - val modules: MutableList, val builtinFunctions: IBuiltinFunctions, val memsizer: IMemSizer): Node { - val namespace = GlobalNamespace(modules, builtinFunctions.names) + private val _modules = mutableListOf() - val mainModule: Module + val modules: List = _modules + val namespace: GlobalNamespace = GlobalNamespace(modules, builtinFunctions.names) + + init { + // insert a container module for all interned strings later + val internedStringsModule = Module(internedStringsModuleName, mutableListOf(), Position.DUMMY, null) + val block = Block(internedStringsModuleName, null, mutableListOf(), true, Position.DUMMY) + internedStringsModule.statements.add(block) + + _modules.add(0, internedStringsModule) + internedStringsModule.linkParents(namespace) // TODO: was .linkParents(this) - probably wrong?! + internedStringsModule.program = this + } + + fun addModule(module: Module): Program { + require(null == _modules.firstOrNull { it.name == module.name }) + { "module '${module.name}' already present" } + _modules.add(0, module) + module.linkParents(namespace) + module.program = this + return this + } + + fun moveModuleToFront(module: Module): Program { + require(_modules.contains(module)) + { "Not a module of this program: '${module.name}'"} + _modules.remove(module) + _modules.add(0, module) + return this + } + + fun allBlocks(): List = modules.flatMap { it.statements.filterIsInstance() } + + fun entrypoint(): Subroutine { + val mainBlocks = allBlocks().filter { it.name=="main" } + return when (mainBlocks.size) { + 0 -> throw FatalAstException("no 'main' block") + 1 -> mainBlocks[0].subScope("start") as Subroutine + else -> throw FatalAstException("more than one 'main' block") + } + } + + val mainModule: Module // TODO: rename Program.mainModule - it's NOT necessarily the one containing the main *block*! get() = modules.first { it.name!=internedStringsModuleName } + val definedLoadAddress: Int get() = mainModule.loadAddress var actualLoadAddress: Int = 0 private val internedStringsUnique = mutableMapOf, List>() - init { - // insert a container module for all interned strings later - if(modules.firstOrNull()?.name != internedStringsModuleName) { - val internedStringsModule = Module(internedStringsModuleName, mutableListOf(), Position.DUMMY, null) - modules.add(0, internedStringsModule) - val block = Block(internedStringsModuleName, null, mutableListOf(), true, Position.DUMMY) - internedStringsModule.statements.add(block) - internedStringsModule.linkParents(this) - internedStringsModule.program = this - } - } - - fun entrypoint(): Subroutine { - val mainBlocks = allBlocks().filter { it.name=="main" } - if(mainBlocks.size > 1) - throw FatalAstException("more than one 'main' block") - if(mainBlocks.isEmpty()) - throw FatalAstException("no 'main' block") - return mainBlocks[0].subScope("start") as Subroutine - } - fun internString(string: StringLiteralValue): List { // Move a string literal into the internal, deduplicated, string pool // replace it with a variable declaration that points to the entry in the pool. @@ -297,10 +318,6 @@ class Program(val name: String, return scopedName } - - - fun allBlocks(): List = modules.flatMap { it.statements.filterIsInstance() } - override val position: Position = Position.DUMMY override var parent: Node get() = throw FatalAstException("program has no parent") @@ -314,10 +331,11 @@ class Program(val name: String, override fun replaceChildNode(node: Node, replacement: Node) { require(node is Module && replacement is Module) - val idx = modules.indexOfFirst { it===node } - modules[idx] = replacement - replacement.parent = this + val idx = _modules.indexOfFirst { it===node } + _modules[idx] = replacement + replacement.parent = this // TODO: why not replacement.program = this; replacement.linkParents(namespace)?! } + } open class Module(override val name: String, @@ -355,7 +373,7 @@ open class Module(override val name: String, } -class GlobalNamespace(val modules: List, private val builtinFunctionNames: Set): Node, INameScope { +class GlobalNamespace(val modules: Iterable, private val builtinFunctionNames: Set): Node, INameScope { override val name = "<<>>" override val position = Position("<<>>", 0, 0, 0) override val statements = mutableListOf() // not used diff --git a/compilerAst/src/prog8/parser/ModuleImporter.kt b/compilerAst/src/prog8/parser/ModuleImporter.kt index 32a2e9646..fbb6ca3f1 100644 --- a/compilerAst/src/prog8/parser/ModuleImporter.kt +++ b/compilerAst/src/prog8/parser/ModuleImporter.kt @@ -51,9 +51,7 @@ class ModuleImporter(private val program: Program, //private fun importModule(stream: CharStream, modulePath: Path, isLibrary: Boolean): Module { private fun importModule(src: SourceCode) : Module { val moduleAst = Prog8Parser.parseModule(src) - moduleAst.program = program - moduleAst.linkParents(program.namespace) - program.modules.add(moduleAst) + program.addModule(moduleAst) // accept additional imports val lines = moduleAst.statements.toMutableList() diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt index 7b6166075..4a27c7463 100644 --- a/compilerAst/test/TestAstToSourceCode.kt +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -18,9 +18,8 @@ import prog8.parser.ParseError class TestAstToSourceCode { private fun generateP8(module: Module) : String { - val program = Program("test", mutableListOf(module), DummyFunctions, DummyMemsizer) - module.linkParents(program) - module.program = program + val program = Program("test", DummyFunctions, DummyMemsizer) + .addModule(module) var generatedText = "" val it = AstToSourceCode({ str -> generatedText += str }, program) diff --git a/compilerAst/test/TestModuleImporter.kt b/compilerAst/test/TestModuleImporter.kt index c35b026ab..0f08b09d5 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compilerAst/test/TestModuleImporter.kt @@ -23,18 +23,16 @@ import kotlin.test.assertContains class TestModuleImporter { private val count = listOf("1st", "2nd", "3rd", "4th", "5th") - lateinit var program: Program + private lateinit var program: Program @BeforeEach fun beforeEach() { - program = Program("foo", mutableListOf(), DummyFunctions, DummyMemsizer) + program = Program("foo", DummyFunctions, DummyMemsizer) } private fun makeImporter(vararg searchIn: String): ModuleImporter = makeImporter(searchIn.asList()) - private fun makeImporter(searchIn: Iterable) = ModuleImporter( - program, - "blah", - searchIn.toList()) + private fun makeImporter(searchIn: Iterable) = + ModuleImporter(program, "blah", searchIn.toList()) @Nested inner class Constructor { @@ -93,11 +91,10 @@ class TestModuleImporter { @Test fun testDirectory() { - val dirRel = assumeDirectory(workingDir.relativize(fixturesDir)) - val searchIn = Path(".", "$dirRel").invariantSeparatorsPathString - val importer = makeImporter(searchIn) - val srcPathRel = dirRel + val srcPathRel = assumeDirectory(workingDir.relativize(fixturesDir)) val srcPathAbs = srcPathRel.absolute() + val searchIn = Path(".", "$srcPathRel").invariantSeparatorsPathString + val importer = makeImporter(searchIn) assertThrows { importer.importModule(srcPathRel) } .let { @@ -207,7 +204,17 @@ class TestModuleImporter { } @Test - fun testImportingFileWithSyntaxError() { + fun testImportingFileWithSyntaxError_once() { + doTestImportingFileWithSyntaxError(1) + } + + @Test + @Disabled("TODO: module that imports faulty module should not be kept in Program.modules") + fun testImportingFileWithSyntaxError_twice() { + doTestImportingFileWithSyntaxError(2) + } + + private fun doTestImportingFileWithSyntaxError(repetitions: Int) { val searchIn = assumeDirectory("./", workingDir.relativize(fixturesDir)) val importer = makeImporter(searchIn.invariantSeparatorsPathString) val importing = assumeReadableFile(fixturesDir, "import_file_with_syntax_error.p8") @@ -215,7 +222,7 @@ class TestModuleImporter { val act = { importer.importModule(importing) } - repeat(2) { n -> + repeat(repetitions) { n -> assertThrows(count[n] + " call") { act() }.let { assertThat(it.position.file, equalTo(imported.absolutePathString())) assertThat("line; should be 1-based", it.position.line, equalTo(2)) diff --git a/compilerAst/test/ast/ProgramTests.kt b/compilerAst/test/ast/ProgramTests.kt new file mode 100644 index 000000000..9020d6936 --- /dev/null +++ b/compilerAst/test/ast/ProgramTests.kt @@ -0,0 +1,118 @@ +package prog8tests + +import prog8tests.helpers.* +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.containsString +import org.hamcrest.Matchers.equalTo +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.* + + +import prog8.ast.Program +import prog8.ast.Module +import prog8.ast.base.Position +import prog8.ast.internedStringsModuleName +import java.lang.IllegalArgumentException +import kotlin.test.assertContains +import kotlin.test.assertNotSame +import kotlin.test.assertSame + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class ProgramTests { + + @Nested + inner class Constructor { + @Test + fun withNameBuiltinsAndMemsizer() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + assertThat(program.modules.size, equalTo(1)) + assertThat(program.modules[0].name, equalTo(internedStringsModuleName)) + assertSame(program, program.modules[0].program) + assertSame(program.namespace, program.modules[0].parent) + } + + } + + @Nested + inner class AddModule { + @Test + fun withEmptyModule() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + val m1 = Module("bar", mutableListOf(), Position.DUMMY, null) + + val retVal = program.addModule(m1) + + assertSame(program, retVal) + assertThat(program.modules.size, equalTo(2)) + assertContains(program.modules, m1) + assertSame(program, m1.program) + assertSame(program.namespace, m1.parent) + + assertThrows { program.addModule(m1) } + .let { assertThat(it.message, containsString(m1.name)) } + + val m2 = Module(m1.name, mutableListOf(), m1.position, m1.source) + assertThrows { program.addModule(m2) } + .let { assertThat(it.message, containsString(m2.name)) } + } + } + + @Nested + inner class MoveModuleToFront { + @Test + fun withInternedStringsModule() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + val m = program.modules[0] + assertThat(m.name, equalTo(internedStringsModuleName)) + + val retVal = program.moveModuleToFront(m) + assertSame(program, retVal) + assertSame(m, program.modules[0]) + } + @Test + fun withForeignModule() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + val m = Module("bar", mutableListOf(), Position.DUMMY, null) + + assertThrows { program.moveModuleToFront(m) } + } + @Test + fun withFirstOfPreviouslyAddedModules() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + val m1 = Module("bar", mutableListOf(), Position.DUMMY, null) + val m2 = Module("qmbl", mutableListOf(), Position.DUMMY, null) + program.addModule(m1) + program.addModule(m2) + + val retVal = program.moveModuleToFront(m1) + assertSame(program, retVal) + assertThat(program.modules.indexOf(m1), equalTo(0)) + } + @Test + fun withSecondOfPreviouslyAddedModules() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + val m1 = Module("bar", mutableListOf(), Position.DUMMY, null) + val m2 = Module("qmbl", mutableListOf(), Position.DUMMY, null) + program.addModule(m1) + program.addModule(m2) + + val retVal = program.moveModuleToFront(m2) + assertSame(program, retVal) + assertThat(program.modules.indexOf(m2), equalTo(0)) + } + } + + @Nested + inner class Properties { + @Test + fun modules() { + val program = Program("foo", DummyFunctions, DummyMemsizer) + + val ms1 = program.modules + val ms2 = program.modules + assertSame(ms1, ms2) + } + } +} From fb67d1155fd6808beb430c8e27e612e1cc99f38c Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 2 Aug 2021 08:57:09 +0200 Subject: [PATCH 54/68] * structure TestCompilerOnImportsAndIncludes, add (@Disabled for now) test re %import with string arg --- .../test/TestCompilerOnImportsAndIncludes.kt | 208 ++++++++++-------- .../fixtures/importFromSameFolder_strLit.p8 | 9 + 2 files changed, 129 insertions(+), 88 deletions(-) create mode 100644 compiler/test/fixtures/importFromSameFolder_strLit.p8 diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index f08259c11..904f0fd3d 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -1,14 +1,15 @@ package prog8tests +import prog8tests.helpers.* import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.Test -import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestFactory +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.DynamicTest import org.junit.jupiter.api.DynamicTest.dynamicTest import kotlin.test.* import kotlin.io.path.* -import prog8tests.helpers.* import prog8.ast.expressions.AddressOf import prog8.ast.expressions.IdentifierReference @@ -26,97 +27,128 @@ import prog8.compiler.target.Cx16Target @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestCompilerOnImportsAndIncludes { - @Test - fun testImportFromSameFolder() { - val filepath = assumeReadableFile(fixturesDir, "importFromSameFolder.p8") - assumeReadableFile(fixturesDir, "foo_bar.p8") + @Nested + inner class Import { - val platform = Cx16Target - val result = compileFile(platform, false, fixturesDir, filepath.name) - .assertSuccess() + @Test + fun testImportFromSameFolder() { + val filepath = assumeReadableFile(fixturesDir, "importFromSameFolder.p8") + assumeReadableFile(fixturesDir, "foo_bar.p8") - val program = result.programAst - val startSub = program.entrypoint() - val strLits = startSub.statements - .filterIsInstance() - .map { it.args[0] as IdentifierReference } - .map { it.targetVarDecl(program)!!.value as StringLiteralValue } + val platform = Cx16Target + val result = compileFile(platform, optimize = false, fixturesDir, filepath.name) + .assertSuccess() - assertEquals("main.bar", strLits[0].value) - assertEquals("foo.bar", strLits[1].value) - assertEquals("main", strLits[0].definingScope().name) - assertEquals("foo", strLits[1].definingScope().name) - } + val program = result.programAst + val startSub = program.entrypoint() + val strLits = startSub.statements + .filterIsInstance() + .map { it.args[0] as IdentifierReference } + .map { it.targetVarDecl(program)!!.value as StringLiteralValue } - @Test - fun testAsmIncludeFromSameFolder() { - val filepath = assumeReadableFile(fixturesDir, "asmIncludeFromSameFolder.p8") - assumeReadableFile(fixturesDir,"foo_bar.asm") - - val platform = Cx16Target - val result = compileFile(platform, false, fixturesDir, filepath.name) - .assertSuccess() - - val program = result.programAst - val startSub = program.entrypoint() - val args = startSub.statements - .filterIsInstance() - .map { it.args[0] } - - val str0 = (args[0] as IdentifierReference).targetVarDecl(program)!!.value as StringLiteralValue - assertEquals("main.bar", str0.value) - assertEquals("main", str0.definingScope().name) - - val id1 = (args[1] as AddressOf).identifier - val lbl1 = id1.targetStatement(program) as Label - assertEquals("foo_bar", lbl1.name) - assertEquals("start", lbl1.definingScope().name) - } - - @Test - fun testAsmbinaryDirectiveWithNonExistingFile() { - val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonExisting.p8") - assumeNotExists(fixturesDir,"i_do_not_exist.bin") - - compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) - .assertFailure() - } - - @Test - fun testAsmbinaryDirectiveWithNonReadableFile() { - val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonReadable.p8") - assumeDirectory(fixturesDir, "subFolder") - - compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) - .assertFailure() - } - - @TestFactory - fun asmbinaryDirectiveWithExistingBinFile(): Iterable = - listOf( - Triple("same ", "asmBinaryFromSameFolder.p8", "do_nothing1.bin"), - Triple("sub", "asmBinaryFromSubFolder.p8", "subFolder/do_nothing2.bin"), - ).map { - val (where, p8Str, binStr) = it - val p8Path = fixturesDir.div(p8Str) // sanity check below, *inside dynamicTest* - val binPath = fixturesDir.div(binStr) - val displayName = "%asmbinary from ${where}folder" - dynamicTest(displayName) { - assumeReadableFile(p8Path) - assumeReadableFile(binPath) - assertNotEquals( // the bug we're testing for (#54) was hidden if outputDir == workinDir - workingDir.normalize().toAbsolutePath(), - outputDir.normalize().toAbsolutePath(), - "sanity check: workingDir and outputDir should not be the same folder") - - compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) - .assertSuccess( - "argument to assembler directive .binary " + - "should be relative to the generated .asm file (in output dir), " + - "NOT relative to .p8 neither current working dir" - ) - } + assertEquals("main.bar", strLits[0].value) + assertEquals("foo.bar", strLits[1].value) + assertEquals("main", strLits[0].definingScope().name) + assertEquals("foo", strLits[1].definingScope().name) } + @Test + @Disabled("TODO: why would we not accept string literals as argument to %import?") + fun testImportFromSameFolder_strLit() { + val filepath = assumeReadableFile(fixturesDir,"importFromSameFolder_strLit.p8") + val imported = assumeReadableFile(fixturesDir, "foo_bar.p8") + + val platform = Cx16Target + val result = compileFile(platform, optimize = false, fixturesDir, filepath.name) + .assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val strLits = startSub.statements + .filterIsInstance() + .map { it.args[0] as IdentifierReference } + .map { it.targetVarDecl(program)!!.value as StringLiteralValue } + + assertEquals("main.bar", strLits[0].value) + assertEquals("foo.bar", strLits[1].value) + assertEquals("main", strLits[0].definingScope().name) + assertEquals("foo", strLits[1].definingScope().name) + } } + @Nested + inner class AsmInclude { + @Test + fun testAsmIncludeFromSameFolder() { + val filepath = assumeReadableFile(fixturesDir, "asmIncludeFromSameFolder.p8") + assumeReadableFile(fixturesDir, "foo_bar.asm") + + val platform = Cx16Target + val result = compileFile(platform, optimize = false, fixturesDir, filepath.name) + .assertSuccess() + + val program = result.programAst + val startSub = program.entrypoint() + val args = startSub.statements + .filterIsInstance() + .map { it.args[0] } + + val str0 = (args[0] as IdentifierReference).targetVarDecl(program)!!.value as StringLiteralValue + assertEquals("main.bar", str0.value) + assertEquals("main", str0.definingScope().name) + + val id1 = (args[1] as AddressOf).identifier + val lbl1 = id1.targetStatement(program) as Label + assertEquals("foo_bar", lbl1.name) + assertEquals("start", lbl1.definingScope().name) + } + } + + @Nested + inner class Asmbinary { + @Test + fun testAsmbinaryDirectiveWithNonExistingFile() { + val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonExisting.p8") + assumeNotExists(fixturesDir, "i_do_not_exist.bin") + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertFailure() + } + + @Test + fun testAsmbinaryDirectiveWithNonReadableFile() { + val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonReadable.p8") + assumeDirectory(fixturesDir, "subFolder") + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertFailure() + } + + @TestFactory + fun asmbinaryDirectiveWithExistingBinFile(): Iterable = + listOf( + Triple("same ", "asmBinaryFromSameFolder.p8", "do_nothing1.bin"), + Triple("sub", "asmBinaryFromSubFolder.p8", "subFolder/do_nothing2.bin"), + ).map { + val (where, p8Str, binStr) = it + dynamicTest("%asmbinary from ${where}folder") { + val p8Path = assumeReadableFile(fixturesDir, p8Str) + val binPath = assumeReadableFile(fixturesDir, binStr) + assertNotEquals( // the bug we're testing for (#54) was hidden if outputDir == workinDir + workingDir.normalize().toAbsolutePath(), + outputDir.normalize().toAbsolutePath(), + "sanity check: workingDir and outputDir should not be the same folder" + ) + + compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) + .assertSuccess( + "argument to assembler directive .binary " + + "should be relative to the generated .asm file (in output dir), " + + "NOT relative to .p8 neither current working dir" + ) + } + } + + } + +} \ No newline at end of file diff --git a/compiler/test/fixtures/importFromSameFolder_strLit.p8 b/compiler/test/fixtures/importFromSameFolder_strLit.p8 new file mode 100644 index 000000000..0f75c437f --- /dev/null +++ b/compiler/test/fixtures/importFromSameFolder_strLit.p8 @@ -0,0 +1,9 @@ +%import textio +%import "foo_bar.p8" +main { + str myBar = "main.bar" + sub start() { + txt.print(myBar) + txt.print(foo.bar) + } +} From ac02a9993448ed7f5f5e869aa28b9527cd6748ad Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 2 Aug 2021 09:15:11 +0200 Subject: [PATCH 55/68] * move ModuleImporter to prog8.compiler (package & module), together with its tests --- compiler/src/prog8/compiler/Compiler.kt | 1 - .../parser => compiler/src/prog8/compiler}/ModuleImporter.kt | 4 +++- .../test/ModuleImporterTests.kt | 2 +- compiler/test/fixtures/file_with_syntax_error.p8 | 2 ++ .../test/fixtures/import_file_with_syntax_error.p8 | 0 .../test/fixtures/import_import_nonexisting.p8 | 0 {compilerAst => compiler}/test/fixtures/import_nonexisting.p8 | 0 7 files changed, 6 insertions(+), 3 deletions(-) rename {compilerAst/src/prog8/parser => compiler/src/prog8/compiler}/ModuleImporter.kt (98%) rename compilerAst/test/TestModuleImporter.kt => compiler/test/ModuleImporterTests.kt (99%) create mode 100644 compiler/test/fixtures/file_with_syntax_error.p8 rename {compilerAst => compiler}/test/fixtures/import_file_with_syntax_error.p8 (100%) rename {compilerAst => compiler}/test/fixtures/import_import_nonexisting.p8 (100%) rename {compilerAst => compiler}/test/fixtures/import_nonexisting.p8 (100%) diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index 557b23a2a..e0ae421e1 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -16,7 +16,6 @@ import prog8.compiler.target.Cx16Target import prog8.compiler.target.ICompilationTarget import prog8.compiler.target.asmGeneratorFor import prog8.optimizer.* -import prog8.parser.ModuleImporter import prog8.parser.ParsingFailedError import java.io.File import java.io.InputStream diff --git a/compilerAst/src/prog8/parser/ModuleImporter.kt b/compiler/src/prog8/compiler/ModuleImporter.kt similarity index 98% rename from compilerAst/src/prog8/parser/ModuleImporter.kt rename to compiler/src/prog8/compiler/ModuleImporter.kt index fbb6ca3f1..c18e32cc3 100644 --- a/compilerAst/src/prog8/parser/ModuleImporter.kt +++ b/compiler/src/prog8/compiler/ModuleImporter.kt @@ -1,4 +1,4 @@ -package prog8.parser +package prog8.compiler import prog8.ast.Module import prog8.ast.Program @@ -6,6 +6,8 @@ import prog8.ast.base.Position import prog8.ast.base.SyntaxError import prog8.ast.statements.Directive import prog8.ast.statements.DirectiveArg +import prog8.parser.Prog8Parser +import prog8.parser.SourceCode import java.io.File import java.nio.file.Path import kotlin.io.FileSystemException diff --git a/compilerAst/test/TestModuleImporter.kt b/compiler/test/ModuleImporterTests.kt similarity index 99% rename from compilerAst/test/TestModuleImporter.kt rename to compiler/test/ModuleImporterTests.kt index 0f08b09d5..19cef532a 100644 --- a/compilerAst/test/TestModuleImporter.kt +++ b/compiler/test/ModuleImporterTests.kt @@ -15,7 +15,7 @@ import kotlin.io.path.* import prog8.ast.Program import prog8.parser.ParseError -import prog8.parser.ModuleImporter +import prog8.compiler.ModuleImporter import kotlin.test.assertContains diff --git a/compiler/test/fixtures/file_with_syntax_error.p8 b/compiler/test/fixtures/file_with_syntax_error.p8 new file mode 100644 index 000000000..6b9930fe0 --- /dev/null +++ b/compiler/test/fixtures/file_with_syntax_error.p8 @@ -0,0 +1,2 @@ +; test expects the following 2nd (!) line: +bad { } ; -> missing EOL at '}' (ie: *after* the '{') diff --git a/compilerAst/test/fixtures/import_file_with_syntax_error.p8 b/compiler/test/fixtures/import_file_with_syntax_error.p8 similarity index 100% rename from compilerAst/test/fixtures/import_file_with_syntax_error.p8 rename to compiler/test/fixtures/import_file_with_syntax_error.p8 diff --git a/compilerAst/test/fixtures/import_import_nonexisting.p8 b/compiler/test/fixtures/import_import_nonexisting.p8 similarity index 100% rename from compilerAst/test/fixtures/import_import_nonexisting.p8 rename to compiler/test/fixtures/import_import_nonexisting.p8 diff --git a/compilerAst/test/fixtures/import_nonexisting.p8 b/compiler/test/fixtures/import_nonexisting.p8 similarity index 100% rename from compilerAst/test/fixtures/import_nonexisting.p8 rename to compiler/test/fixtures/import_nonexisting.p8 From c2986eaf477b10c111d089e281545f8baaf299a5 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 2 Aug 2021 14:52:46 +0200 Subject: [PATCH 56/68] * structure TestProg8Parser with @Nested --- compilerAst/test/TestProg8Parser.kt | 842 +++++++++++++++------------- 1 file changed, 451 insertions(+), 391 deletions(-) diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index 55e7da561..f2d53a11f 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -4,6 +4,7 @@ import prog8tests.helpers.* import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.Test import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Nested import kotlin.test.* import kotlin.io.path.* @@ -19,452 +20,511 @@ import prog8.ast.expressions.* @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestProg8Parser { - @Test - fun testModuleSourceNeedNotEndWithNewline() { - val nl = "\n" // say, Unix-style (different flavours tested elsewhere) - val src = SourceCode.of("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) + @Nested + inner class Newline { - // #45: Prog8ANTLRParser would report (throw) "missing at ''" - val module = parseModule(src) - assertEquals(1, module.statements.size) - } + @Nested + inner class AtEnd { - @Test - fun testModuleSourceMayEndWithNewline() { - val nl = "\n" // say, Unix-style (different flavours tested elsewhere) - val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) - val module = parseModule(SourceCode.of(srcText)) - assertEquals(1, module.statements.size) - } + @Test + fun testModuleSourceNeedNotEndWithNewline() { + val nl = "\n" // say, Unix-style (different flavours tested elsewhere) + val src = SourceCode.of("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) - @Test - fun testAllBlocksButLastMustEndWithNewline() { - val nl = "\n" // say, Unix-style (different flavours tested elsewhere) - - // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end - val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl - - // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed - val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" - - assertFailsWith { parseModule(SourceCode.of(srcBad)) } - val module = parseModule(SourceCode.of(srcGood)) - assertEquals(2, module.statements.size) - } - - @Test - fun testWindowsAndMacNewlinesAreAlsoFine() { - val nlWin = "\r\n" - val nlUnix = "\n" - val nlMac = "\r" - - //parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath()) - - // a good mix of all kinds of newlines: - val srcText = - "foo {" + - nlMac + - nlWin + - "}" + - nlMac + // <-- do test a single \r (!) where an EOL is expected - "bar {" + - nlUnix + - "}" + - nlUnix + nlMac // both should be "eaten up" by just one EOL token - "combi {" + - nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token - "}" + - nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) - - val module = parseModule(SourceCode.of(srcText)) - assertEquals(2, module.statements.size) - } - - @Test - fun testInterleavedEolAndCommentBeforeFirstBlock() { - // issue: #47 - val srcText = """ - ; comment - - ; comment - - blockA { + // #45: Prog8ANTLRParser would report (throw) "missing at ''" + val module = parseModule(src) + assertEquals(1, module.statements.size) } -""" - val module = parseModule(SourceCode.of(srcText)) - assertEquals(1, module.statements.size) - } - @Test - fun testInterleavedEolAndCommentBetweenBlocks() { - // issue: #47 - val srcText = """ - blockA { + @Test + fun testModuleSourceMayEndWithNewline() { + val nl = "\n" // say, Unix-style (different flavours tested elsewhere) + val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) + val module = parseModule(SourceCode.of(srcText)) + assertEquals(1, module.statements.size) } - ; comment - - ; comment - - blockB { - } -""" - val module = parseModule(SourceCode.of(srcText)) - assertEquals(2, module.statements.size) - } + } - @Test - fun testInterleavedEolAndCommentAfterLastBlock() { - // issue: #47 - val srcText = """ - blockA { - } - ; comment - - ; comment - -""" - val module = parseModule(SourceCode.of(srcText)) - assertEquals(1, module.statements.size) - } + @Test + fun testAllBlocksButLastMustEndWithNewline() { + val nl = "\n" // say, Unix-style (different flavours tested elsewhere) - @Test - fun testNewlineBetweenTwoBlocksOrDirectivesStillRequired() { - // issue: #47 + // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end + val srcBad = "foo {" + nl + "}" + " bar {" + nl + "}" + nl - // block and block - assertFailsWith{ parseModule(SourceCode.of(""" - blockA { - } blockB { - } - """)) } + // GOOD: 2nd block `bar` does start on a new line; however, a nl at the very end ain't needed + val srcGood = "foo {" + nl + "}" + nl + "bar {" + nl + "}" - // block and directive - assertFailsWith{ parseModule(SourceCode.of(""" - blockB { - } %import textio - """)) } + assertFailsWith { parseModule(SourceCode.of(srcBad)) } + val module = parseModule(SourceCode.of(srcGood)) + assertEquals(2, module.statements.size) + } - // The following two are bogus due to directive *args* expected to follow the directive name. - // Leaving them in anyways. + @Test + fun testNewlineBetweenTwoBlocksOrDirectivesStillRequired() { + // issue: #47 - // dir and block - assertFailsWith{ parseModule(SourceCode.of(""" - %import textio blockB { - } - """)) } + // block and block + assertFailsWith{ parseModule(SourceCode.of(""" + blockA { + } blockB { + } + """)) } - assertFailsWith{ parseModule(SourceCode.of(""" - %import textio %import syslib - """)) } - } + // block and directive + assertFailsWith{ parseModule(SourceCode.of(""" + blockB { + } %import textio + """)) } - @Test - fun parseModuleShouldNotLookAtImports() { - val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") - assumeNotExists(fixturesDir, "i_do_not_exist.p8") - val text = "%import ${importedNoExt.name}" - val module = parseModule(SourceCode.of(text)) + // The following two are bogus due to directive *args* expected to follow the directive name. + // Leaving them in anyways. - assertEquals(1, module.statements.size) - } + // dir and block + assertFailsWith{ parseModule(SourceCode.of(""" + %import textio blockB { + } + """)) } + assertFailsWith{ parseModule(SourceCode.of(""" + %import textio %import syslib + """)) } + } - @Test - fun testParseModuleWithEmptyString() { - val module = parseModule(SourceCode.of("")) - assertEquals(0, module.statements.size) - } + @Test + fun testWindowsAndMacNewlinesAreAlsoFine() { + val nlWin = "\r\n" + val nlUnix = "\n" + val nlMac = "\r" - @Test - fun testParseModuleWithEmptyFile() { - val path = assumeReadableFile(fixturesDir,"empty.p8") - val module = parseModule(SourceCode.fromPath(path)) - assertEquals(0, module.statements.size) - } + //parseModule(Paths.get("test", "fixtures", "mac_newlines.p8").toAbsolutePath()) - @Test - fun testModuleNameForSourceFromString() { - val srcText = """ - main { - } - """.trimIndent() - val module = parseModule(SourceCode.of(srcText)) + // a good mix of all kinds of newlines: + val srcText = + "foo {" + + nlMac + + nlWin + + "}" + + nlMac + // <-- do test a single \r (!) where an EOL is expected + "bar {" + + nlUnix + + "}" + + nlUnix + nlMac // both should be "eaten up" by just one EOL token + "combi {" + + nlMac + nlWin + nlUnix // all three should be "eaten up" by just one EOL token + "}" + + nlUnix // end with newline (see testModuleSourceNeedNotEndWithNewline) - // Note: assertContains has *actual* as first param - assertContains(module.name, Regex("^anonymous_[0-9a-f]+$")) - } - - @Test - fun testModuleNameForSourceFromPath() { - val path = assumeReadableFile(fixturesDir,"simple_main.p8") - val module = parseModule(SourceCode.fromPath(path)) - assertEquals(path.nameWithoutExtension, module.name) - } - - - fun assertPosition(actual: Position, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) { - require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) - if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") - if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) - if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") - if (expFile != null) assertEquals(expFile, actual.file, ".position.file") - } - - fun assertPosition(actual: Position, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) { - require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) - if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") - if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)" ) - if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") - // Note: assertContains expects *actual* value first - if (expFile != null) assertContains(actual.file, expFile, ".position.file") - } - - fun assertPositionOf(actual: Node, expFile: String? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) = - assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) - - fun assertPositionOf(actual: Node, expFile: Regex? = null, expLine: Int? = null, expStartCol: Int? = null, expEndCol: Int? = null) = - assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) - - - @Test - fun testErrorLocationForSourceFromString() { - val srcText = "bad * { }\n" - - assertFailsWith { parseModule(SourceCode.of(srcText)) } - try { - parseModule(SourceCode.of(srcText)) - } catch (e: ParseError) { - assertPosition(e.position, Regex("^$"), 1, 4, 4) + val module = parseModule(SourceCode.of(srcText)) + assertEquals(2, module.statements.size) } } - @Test - fun testErrorLocationForSourceFromPath() { - val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") + @Nested + inner class EOLsInterleavedWithComments { - assertFailsWith { parseModule(SourceCode.fromPath(path)) } - try { - parseModule(SourceCode.fromPath(path)) - } catch (e: ParseError) { - assertPosition(e.position, path.absolutePathString(), 2, 6) // TODO: endCol wrong + @Test + fun testInterleavedEolAndCommentBeforeFirstBlock() { + // issue: #47 + val srcText = """ + ; comment + + ; comment + + blockA { + } + """ + val module = parseModule(SourceCode.of(srcText)) + assertEquals(1, module.statements.size) + } + + @Test + fun testInterleavedEolAndCommentBetweenBlocks() { + // issue: #47 + val srcText = """ + blockA { + } + ; comment + + ; comment + + blockB { + } + """ + val module = parseModule(SourceCode.of(srcText)) + assertEquals(2, module.statements.size) + } + + @Test + fun testInterleavedEolAndCommentAfterLastBlock() { + // issue: #47 + val srcText = """ + blockA { + } + ; comment + + ; comment + + """ + val module = parseModule(SourceCode.of(srcText)) + assertEquals(1, module.statements.size) } } - @Test - fun testModulePositionForSourceFromString() { - val srcText = """ - main { + + @Nested + inner class ImportDirectives { + @Test + fun parseModuleShouldNotLookAtImports() { + val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") + assumeNotExists(fixturesDir, "i_do_not_exist.p8") + val text = "%import ${importedNoExt.name}" + val module = parseModule(SourceCode.of(text)) + + assertEquals(1, module.statements.size) + } + } + + + @Nested + inner class EmptySourcecode { + @Test + fun testParseModuleWithEmptyString() { + val module = parseModule(SourceCode.of("")) + assertEquals(0, module.statements.size) + } + + @Test + fun testParseModuleWithEmptyFile() { + val path = assumeReadableFile(fixturesDir, "empty.p8") + val module = parseModule(SourceCode.fromPath(path)) + assertEquals(0, module.statements.size) + } + } + + @Nested + inner class NameOfModule { + @Test + fun testModuleNameForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + + // Note: assertContains has *actual* as first param + assertContains(module.name, Regex("^anonymous_[0-9a-f]+$")) + } + + @Test + fun testModuleNameForSourceFromPath() { + val path = assumeReadableFile(fixturesDir, "simple_main.p8") + val module = parseModule(SourceCode.fromPath(path)) + assertEquals(path.nameWithoutExtension, module.name) + } + } + + @Nested + inner class PositionOfAstNodesAndParseErrors { + + + fun assertPosition( + actual: Position, + expFile: String? = null, + expLine: Int? = null, + expStartCol: Int? = null, + expEndCol: Int? = null + ) { + require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) + if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") + if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)") + if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + if (expFile != null) assertEquals(expFile, actual.file, ".position.file") + } + + fun assertPosition( + actual: Position, + expFile: Regex? = null, + expLine: Int? = null, + expStartCol: Int? = null, + expEndCol: Int? = null + ) { + require(!listOf(expLine, expStartCol, expEndCol).all { it == null }) + if (expLine != null) assertEquals(expLine, actual.line, ".position.line (1-based)") + if (expStartCol != null) assertEquals(expStartCol, actual.startCol, ".position.startCol (0-based)") + if (expEndCol != null) assertEquals(expEndCol, actual.endCol, ".position.endCol (0-based)") + // Note: assertContains expects *actual* value first + if (expFile != null) assertContains(actual.file, expFile, ".position.file") + } + + fun assertPositionOf( + actual: Node, + expFile: String? = null, + expLine: Int? = null, + expStartCol: Int? = null, + expEndCol: Int? = null + ) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + fun assertPositionOf( + actual: Node, + expFile: Regex? = null, + expLine: Int? = null, + expStartCol: Int? = null, + expEndCol: Int? = null + ) = + assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) + + + @Test + fun testErrorLocationForSourceFromString() { + val srcText = "bad * { }\n" + + assertFailsWith { parseModule(SourceCode.of(srcText)) } + try { + parseModule(SourceCode.of(srcText)) + } catch (e: ParseError) { + assertPosition(e.position, Regex("^$"), 1, 4, 4) } - """.trimIndent() - val module = parseModule(SourceCode.of(srcText)) - assertPositionOf(module, Regex("^$"), 1, 0) // TODO: endCol wrong - } + } - @Test - fun testModulePositionForSourceFromPath() { - val path = assumeReadableFile(fixturesDir,"simple_main.p8") + @Test + fun testErrorLocationForSourceFromPath() { + val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") - val module = parseModule(SourceCode.fromPath(path)) - assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong - } + assertFailsWith { parseModule(SourceCode.fromPath(path)) } + try { + parseModule(SourceCode.fromPath(path)) + } catch (e: ParseError) { + assertPosition(e.position, path.absolutePathString(), 2, 6) // TODO: endCol wrong + } + } - @Test - fun testInnerNodePositionsForSourceFromPath() { - val path = assumeReadableFile(fixturesDir,"simple_main.p8") + @Test + fun testModulePositionForSourceFromString() { + val srcText = """ + main { + } + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + assertPositionOf(module, Regex("^$"), 1, 0) // TODO: endCol wrong + } - val module = parseModule(SourceCode.fromPath(path)) - val mpf = module.position.file + @Test + fun testModulePositionForSourceFromPath() { + val path = assumeReadableFile(fixturesDir, "simple_main.p8") - assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong - val mainBlock = module.statements.filterIsInstance()[0] - assertPositionOf(mainBlock, mpf, 1, 0) // TODO: endCol wrong! - val startSub = mainBlock.statements.filterIsInstance()[0] - assertPositionOf(startSub, mpf, 2, 4) // TODO: endCol wrong! - } + val module = parseModule(SourceCode.fromPath(path)) + assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong + } - /** - * TODO: this test is testing way too much at once - */ - @Test - @Disabled("TODO: fix .position of nodes below Module - step 8, 'refactor AST gen'") - fun testInnerNodePositionsForSourceFromString() { - val srcText = """ - %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? - main { - sub start() { - ubyte foo = 42 - ubyte bar - when (foo) { - 23 -> bar = 'x' ; WhenChoice, also directly inheriting Node - 42 -> bar = 'y' - else -> bar = 'z' + @Test + fun testInnerNodePositionsForSourceFromPath() { + val path = assumeReadableFile(fixturesDir, "simple_main.p8") + + val module = parseModule(SourceCode.fromPath(path)) + val mpf = module.position.file + + assertPositionOf(module, path.absolutePathString(), 1, 0) // TODO: endCol wrong + val mainBlock = module.statements.filterIsInstance()[0] + assertPositionOf(mainBlock, mpf, 1, 0) // TODO: endCol wrong! + val startSub = mainBlock.statements.filterIsInstance()[0] + assertPositionOf(startSub, mpf, 2, 4) // TODO: endCol wrong! + } + + + /** + * TODO: this test is testing way too much at once + */ + @Test + @Disabled("TODO: fix .position of nodes below Module - step 8, 'refactor AST gen'") + fun testInnerNodePositionsForSourceFromString() { + val srcText = """ + %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? + main { + sub start() { + ubyte foo = 42 + ubyte bar + when (foo) { + 23 -> bar = 'x' ; WhenChoice, also directly inheriting Node + 42 -> bar = 'y' + else -> bar = 'z' + } } } - } - """.trimIndent() - val module = parseModule(SourceCode.of(srcText)) - val mpf = module.position.file + """.trimIndent() + val module = parseModule(SourceCode.of(srcText)) + val mpf = module.position.file - val targetDirective = module.statements.filterIsInstance()[0] - assertPositionOf(targetDirective, mpf, 1, 0) // TODO: endCol wrong! - val mainBlock = module.statements.filterIsInstance()[0] - assertPositionOf(mainBlock, mpf, 2, 0) // TODO: endCol wrong! - val startSub = mainBlock.statements.filterIsInstance()[0] - assertPositionOf(startSub, mpf, 3, 4) // TODO: endCol wrong! - val declFoo = startSub.statements.filterIsInstance()[0] - assertPositionOf(declFoo, mpf, 4, 8) // TODO: endCol wrong! - val rhsFoo = declFoo.value!! - assertPositionOf(rhsFoo, mpf, 4, 20) // TODO: endCol wrong! - val declBar = startSub.statements.filterIsInstance()[1] - assertPositionOf(declBar, mpf, 5, 8) // TODO: endCol wrong! - val whenStmt = startSub.statements.filterIsInstance()[0] - assertPositionOf(whenStmt, mpf, 6, 8) // TODO: endCol wrong! - assertPositionOf(whenStmt.choices[0], mpf, 7, 12) // TODO: endCol wrong! - assertPositionOf(whenStmt.choices[1], mpf, 8, 12) // TODO: endCol wrong! - assertPositionOf(whenStmt.choices[2], mpf, 9, 12) // TODO: endCol wrong! + val targetDirective = module.statements.filterIsInstance()[0] + assertPositionOf(targetDirective, mpf, 1, 0) // TODO: endCol wrong! + val mainBlock = module.statements.filterIsInstance()[0] + assertPositionOf(mainBlock, mpf, 2, 0) // TODO: endCol wrong! + val startSub = mainBlock.statements.filterIsInstance()[0] + assertPositionOf(startSub, mpf, 3, 4) // TODO: endCol wrong! + val declFoo = startSub.statements.filterIsInstance()[0] + assertPositionOf(declFoo, mpf, 4, 8) // TODO: endCol wrong! + val rhsFoo = declFoo.value!! + assertPositionOf(rhsFoo, mpf, 4, 20) // TODO: endCol wrong! + val declBar = startSub.statements.filterIsInstance()[1] + assertPositionOf(declBar, mpf, 5, 8) // TODO: endCol wrong! + val whenStmt = startSub.statements.filterIsInstance()[0] + assertPositionOf(whenStmt, mpf, 6, 8) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[0], mpf, 7, 12) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[1], mpf, 8, 12) // TODO: endCol wrong! + assertPositionOf(whenStmt.choices[2], mpf, 9, 12) // TODO: endCol wrong! + } } - @Test - fun testCharLitAsArg() { - val src = SourceCode.of(""" - main { - sub start() { - chrout('\n') + @Nested + inner class CharLiterals { + + @Test + fun testCharLitAsArg() { + val src = SourceCode.of(""" + main { + sub start() { + chrout('\n') + } } - } - """) - val module = parseModule(src) + """) + val module = parseModule(src) - val startSub = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - val funCall = startSub.statements.filterIsInstance().first() + val startSub = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + val funCall = startSub.statements.filterIsInstance().first() - assertIs(funCall.args[0]) - val char = funCall.args[0] as CharLiteral - assertEquals('\n', char.value) - } + assertIs(funCall.args[0]) + val char = funCall.args[0] as CharLiteral + assertEquals('\n', char.value) + } - @Test - fun testBlockLevelVarDeclWithCharLiteral_noAltEnc() { - val src = SourceCode.of(""" - main { - ubyte c = 'x' - } - """) - val module = parseModule(src) - val decl = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - - val rhs = decl.value as CharLiteral - assertEquals('x', rhs.value, "char literal's .value") - assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") - } - - @Test - fun testBlockLevelConstDeclWithCharLiteral_withAltEnc() { - val src = SourceCode.of(""" - main { - const ubyte c = @'x' - } - """) - val module = parseModule(src) - val decl = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - - val rhs = decl.value as CharLiteral - assertEquals('x', rhs.value, "char literal's .value") - assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") - } - - @Test - fun testSubRoutineLevelVarDeclWithCharLiteral_noAltEnc() { - val src = SourceCode.of(""" - main { - sub start() { + @Test + fun testBlockLevelVarDeclWithCharLiteral_noAltEnc() { + val src = SourceCode.of(""" + main { ubyte c = 'x' } - } - """) - val module = parseModule(src) - val decl = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] - val rhs = decl.value as CharLiteral - assertEquals('x', rhs.value, "char literal's .value") - assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") - } + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") + } - @Test - fun testSubRoutineLevelConstDeclWithCharLiteral_withAltEnc() { - val src = SourceCode.of(""" - main { - sub start() { + @Test + fun testBlockLevelConstDeclWithCharLiteral_withAltEnc() { + val src = SourceCode.of(""" + main { const ubyte c = @'x' } - } - """) - val module = parseModule(src) - val decl = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] - val rhs = decl.value as CharLiteral - assertEquals('x', rhs.value, "char literal's .value") - assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") - } + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") + } - - @Test - fun testForloop() { - val module = parseModule(SourceCode.of(""" - main { - sub start() { - ubyte ub - for ub in "start" downto "end" { ; #0 - } - for ub in "something" { ; #1 - } - for ub in @'a' to 'f' { ; #2 - } - for ub in false to true { ; #3 - } - for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check! + @Test + fun testSubRoutineLevelVarDeclWithCharLiteral_noAltEnc() { + val src = SourceCode.of(""" + main { + sub start() { + ubyte c = 'x' } } - } - """)) - val iterables = module - .statements.filterIsInstance()[0] - .statements.filterIsInstance()[0] - .statements.filterIsInstance() - .map { it.iterable } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] - assertEquals(5, iterables.size) + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(false, rhs.altEncoding, "char literal's .altEncoding") + } - val it0 = iterables[0] as RangeExpr - assertIs(it0.from, "parser should leave it as is") - assertIs(it0.to, "parser should leave it as is") + @Test + fun testSubRoutineLevelConstDeclWithCharLiteral_withAltEnc() { + val src = SourceCode.of(""" + main { + sub start() { + const ubyte c = @'x' + } + } + """) + val module = parseModule(src) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] - val it1 = iterables[1] as StringLiteralValue - assertEquals("something", it1.value, "parser should leave it as is") - - val it2 = iterables[2] as RangeExpr - assertIs(it2.from, "parser should leave it as is") - assertIs(it2.to, "parser should leave it as is") - - val it3 = iterables[3] as RangeExpr - // TODO: intro BoolLiteral - assertIs(it3.from, "parser should leave it as is") - assertIs(it3.to, "parser should leave it as is") - - val it4 = iterables[4] as RangeExpr - assertIs(it4.from, "parser should leave it as is") - assertIs(it4.to, "parser should leave it as is") + val rhs = decl.value as CharLiteral + assertEquals('x', rhs.value, "char literal's .value") + assertEquals(true, rhs.altEncoding, "char literal's .altEncoding") + } } + + @Nested + inner class Ranges { + + @Test + fun testForloop() { + val module = parseModule(SourceCode.of(""" + main { + sub start() { + ubyte ub + for ub in "start" downto "end" { ; #0 + } + for ub in "something" { ; #1 + } + for ub in @'a' to 'f' { ; #2 + } + for ub in false to true { ; #3 + } + for ub in 9 to 1 { ; #4 - yes, *parser* should NOT check! + } + } + } + """)) + val iterables = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + .statements.filterIsInstance() + .map { it.iterable } + + assertEquals(5, iterables.size) + + val it0 = iterables[0] as RangeExpr + assertIs(it0.from, "parser should leave it as is") + assertIs(it0.to, "parser should leave it as is") + + val it1 = iterables[1] as StringLiteralValue + assertEquals("something", it1.value, "parser should leave it as is") + + val it2 = iterables[2] as RangeExpr + assertIs(it2.from, "parser should leave it as is") + assertIs(it2.to, "parser should leave it as is") + + val it3 = iterables[3] as RangeExpr + // TODO: intro BoolLiteral + assertIs(it3.from, "parser should leave it as is") + assertIs(it3.to, "parser should leave it as is") + + val it4 = iterables[4] as RangeExpr + assertIs(it4.from, "parser should leave it as is") + assertIs(it4.to, "parser should leave it as is") + } + } + } From f0c150d93b30ba96f2ceaa23ddfc6d94012050f5 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 2 Aug 2021 15:36:08 +0200 Subject: [PATCH 57/68] * improve test method names in TestProg8Parser by means of `backtick syntax` --- compilerAst/test/TestProg8Parser.kt | 63 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/compilerAst/test/TestProg8Parser.kt b/compilerAst/test/TestProg8Parser.kt index f2d53a11f..d30abcf23 100644 --- a/compilerAst/test/TestProg8Parser.kt +++ b/compilerAst/test/TestProg8Parser.kt @@ -27,17 +27,17 @@ class TestProg8Parser { inner class AtEnd { @Test - fun testModuleSourceNeedNotEndWithNewline() { + fun `is not required - #40, fixed by #45`() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val src = SourceCode.of("foo {" + nl + "}") // source ends with '}' (= NO newline, issue #40) - // #45: Prog8ANTLRParser would report (throw) "missing at ''" + // #40: Prog8ANTLRParser would report (throw) "missing at ''" val module = parseModule(src) assertEquals(1, module.statements.size) } @Test - fun testModuleSourceMayEndWithNewline() { + fun `is still accepted - #40, fixed by #45`() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) val srcText = "foo {" + nl + "}" + nl // source does end with a newline (issue #40) val module = parseModule(SourceCode.of(srcText)) @@ -46,7 +46,7 @@ class TestProg8Parser { } @Test - fun testAllBlocksButLastMustEndWithNewline() { + fun `is required after each block except the last`() { val nl = "\n" // say, Unix-style (different flavours tested elsewhere) // BAD: 2nd block `bar` does NOT start on new line; however, there's is a nl at the very end @@ -61,9 +61,7 @@ class TestProg8Parser { } @Test - fun testNewlineBetweenTwoBlocksOrDirectivesStillRequired() { - // issue: #47 - + fun `is required between two Blocks or Directives - #47`() { // block and block assertFailsWith{ parseModule(SourceCode.of(""" blockA { @@ -92,7 +90,7 @@ class TestProg8Parser { } @Test - fun testWindowsAndMacNewlinesAreAlsoFine() { + fun `can be Win, Unix or mixed, even mixed`() { val nlWin = "\r\n" val nlUnix = "\n" val nlMac = "\r" @@ -124,7 +122,7 @@ class TestProg8Parser { inner class EOLsInterleavedWithComments { @Test - fun testInterleavedEolAndCommentBeforeFirstBlock() { + fun `are ok before first block - #47`() { // issue: #47 val srcText = """ ; comment @@ -139,7 +137,7 @@ class TestProg8Parser { } @Test - fun testInterleavedEolAndCommentBetweenBlocks() { + fun `are ok between blocks - #47`() { // issue: #47 val srcText = """ blockA { @@ -156,7 +154,7 @@ class TestProg8Parser { } @Test - fun testInterleavedEolAndCommentAfterLastBlock() { + fun `are ok after last block - #47`() { // issue: #47 val srcText = """ blockA { @@ -175,7 +173,7 @@ class TestProg8Parser { @Nested inner class ImportDirectives { @Test - fun parseModuleShouldNotLookAtImports() { + fun `should not be looked into by the parser`() { val importedNoExt = assumeNotExists(fixturesDir, "i_do_not_exist") assumeNotExists(fixturesDir, "i_do_not_exist.p8") val text = "%import ${importedNoExt.name}" @@ -189,13 +187,13 @@ class TestProg8Parser { @Nested inner class EmptySourcecode { @Test - fun testParseModuleWithEmptyString() { + fun `from an empty string should result in empty Module`() { val module = parseModule(SourceCode.of("")) assertEquals(0, module.statements.size) } @Test - fun testParseModuleWithEmptyFile() { + fun `from an empty file should result in empty Module`() { val path = assumeReadableFile(fixturesDir, "empty.p8") val module = parseModule(SourceCode.fromPath(path)) assertEquals(0, module.statements.size) @@ -205,7 +203,7 @@ class TestProg8Parser { @Nested inner class NameOfModule { @Test - fun testModuleNameForSourceFromString() { + fun `parsed from a string`() { val srcText = """ main { } @@ -217,7 +215,7 @@ class TestProg8Parser { } @Test - fun testModuleNameForSourceFromPath() { + fun `parsed from a file`() { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) assertEquals(path.nameWithoutExtension, module.name) @@ -227,8 +225,7 @@ class TestProg8Parser { @Nested inner class PositionOfAstNodesAndParseErrors { - - fun assertPosition( + private fun assertPosition( actual: Position, expFile: String? = null, expLine: Int? = null, @@ -242,7 +239,7 @@ class TestProg8Parser { if (expFile != null) assertEquals(expFile, actual.file, ".position.file") } - fun assertPosition( + private fun assertPosition( actual: Position, expFile: Regex? = null, expLine: Int? = null, @@ -257,7 +254,7 @@ class TestProg8Parser { if (expFile != null) assertContains(actual.file, expFile, ".position.file") } - fun assertPositionOf( + private fun assertPositionOf( actual: Node, expFile: String? = null, expLine: Int? = null, @@ -266,7 +263,7 @@ class TestProg8Parser { ) = assertPosition(actual.position, expFile, expLine, expStartCol, expEndCol) - fun assertPositionOf( + private fun assertPositionOf( actual: Node, expFile: Regex? = null, expLine: Int? = null, @@ -277,7 +274,7 @@ class TestProg8Parser { @Test - fun testErrorLocationForSourceFromString() { + fun `in ParseError from bad string source code`() { val srcText = "bad * { }\n" assertFailsWith { parseModule(SourceCode.of(srcText)) } @@ -289,7 +286,7 @@ class TestProg8Parser { } @Test - fun testErrorLocationForSourceFromPath() { + fun `in ParseError from bad file source code`() { val path = assumeReadableFile(fixturesDir, "file_with_syntax_error.p8") assertFailsWith { parseModule(SourceCode.fromPath(path)) } @@ -301,7 +298,7 @@ class TestProg8Parser { } @Test - fun testModulePositionForSourceFromString() { + fun `of Module parsed from a string`() { val srcText = """ main { } @@ -311,7 +308,7 @@ class TestProg8Parser { } @Test - fun testModulePositionForSourceFromPath() { + fun `of Module parsed from a file`() { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) @@ -319,7 +316,7 @@ class TestProg8Parser { } @Test - fun testInnerNodePositionsForSourceFromPath() { + fun `of non-root Nodes parsed from file`() { val path = assumeReadableFile(fixturesDir, "simple_main.p8") val module = parseModule(SourceCode.fromPath(path)) @@ -338,7 +335,7 @@ class TestProg8Parser { */ @Test @Disabled("TODO: fix .position of nodes below Module - step 8, 'refactor AST gen'") - fun testInnerNodePositionsForSourceFromString() { + fun `of non-root Nodes parsed from a string`() { val srcText = """ %target 16, "abc" ; DirectiveArg directly inherits from Node - neither an Expression nor a Statement..? main { @@ -380,7 +377,7 @@ class TestProg8Parser { inner class CharLiterals { @Test - fun testCharLitAsArg() { + fun `in argument position, no altEnc`() { val src = SourceCode.of(""" main { sub start() { @@ -401,7 +398,7 @@ class TestProg8Parser { } @Test - fun testBlockLevelVarDeclWithCharLiteral_noAltEnc() { + fun `on rhs of block-level var decl, no AltEnc`() { val src = SourceCode.of(""" main { ubyte c = 'x' @@ -418,7 +415,7 @@ class TestProg8Parser { } @Test - fun testBlockLevelConstDeclWithCharLiteral_withAltEnc() { + fun `on rhs of block-level const decl, with AltEnc`() { val src = SourceCode.of(""" main { const ubyte c = @'x' @@ -435,7 +432,7 @@ class TestProg8Parser { } @Test - fun testSubRoutineLevelVarDeclWithCharLiteral_noAltEnc() { + fun `on rhs of subroutine-level var decl, no AltEnc`() { val src = SourceCode.of(""" main { sub start() { @@ -455,7 +452,7 @@ class TestProg8Parser { } @Test - fun testSubRoutineLevelConstDeclWithCharLiteral_withAltEnc() { + fun `on rhs of subroutine-level const decl, with AltEnc`() { val src = SourceCode.of(""" main { sub start() { @@ -479,7 +476,7 @@ class TestProg8Parser { inner class Ranges { @Test - fun testForloop() { + fun `in for-loops`() { val module = parseModule(SourceCode.of(""" main { sub start() { From bd6c60cf8aa22fd622d279893bf6adad89442076 Mon Sep 17 00:00:00 2001 From: meisl Date: Mon, 2 Aug 2021 15:47:42 +0200 Subject: [PATCH 58/68] * improve test method names in helpers_pathsTests by means of `backtick syntax` --- compilerAst/test/helpers_pathsTests.kt | 80 +++++++++++++------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/compilerAst/test/helpers_pathsTests.kt b/compilerAst/test/helpers_pathsTests.kt index f85fb8363..5d7222283 100644 --- a/compilerAst/test/helpers_pathsTests.kt +++ b/compilerAst/test/helpers_pathsTests.kt @@ -10,7 +10,7 @@ import org.junit.jupiter.api.Nested import org.junit.jupiter.api.assertThrows import kotlin.io.path.* -import kotlin.test.assertContains + // Do not move into folder helpers/! // This folder is also used by compiler/test @@ -26,21 +26,21 @@ class PathsHelpersTests { inner class WithOnePathArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThat("should return the path", assumeNotExists(path), `is`(path)) } @Test - fun testOnExistingFile() { + fun `on existing file`() { assertThrows { assumeNotExists(fixturesDir.div("simple_main.p8")) } } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { assertThrows { assumeNotExists(fixturesDir) } @@ -51,14 +51,14 @@ class PathsHelpersTests { inner class WithOneStringArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThat("should return the path", assumeNotExists("$path"), `is`(path)) } @Test - fun testOnExistingFile() { + fun `on existing file`() { val path = fixturesDir.div("simple_main.p8") assertThrows { assumeNotExists("$path") @@ -66,7 +66,7 @@ class PathsHelpersTests { } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { assertThrows { assumeNotExists("$fixturesDir") } @@ -77,21 +77,21 @@ class PathsHelpersTests { inner class WithPathAndStringArgs { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThat("should return the path", assumeNotExists(fixturesDir, "i_do_not_exist"), `is`(path)) } @Test - fun testOnExistingFile() { + fun `on existing file`() { assertThrows { assumeNotExists(fixturesDir, "simple_main.p8") } } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { assertThrows { assumeNotExists(fixturesDir, "..") } @@ -105,7 +105,7 @@ class PathsHelpersTests { @Nested inner class WithOnePathArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThrows { assumeDirectory(path) @@ -113,7 +113,7 @@ class PathsHelpersTests { } @Test - fun testOnExistingFile() { + fun `on existing file`() { val path = fixturesDir.div("simple_main.p8") assertThrows { assumeDirectory(path) @@ -121,7 +121,7 @@ class PathsHelpersTests { } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { val path = workingDir assertThat("should return the path", assumeDirectory(path), `is`(path)) } @@ -130,7 +130,7 @@ class PathsHelpersTests { @Nested inner class WithOneStringArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThrows { assumeDirectory("$path") @@ -138,7 +138,7 @@ class PathsHelpersTests { } @Test - fun testOnExistingFile() { + fun `on existing file`() { val path = fixturesDir.div("simple_main.p8") assertThrows { assumeDirectory("$path") @@ -146,7 +146,7 @@ class PathsHelpersTests { } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { val path = workingDir assertThat("should return the path", assumeDirectory("$path"), `is`(path)) @@ -156,21 +156,21 @@ class PathsHelpersTests { @Nested inner class WithPathAndStringArgs { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { assertThrows { assumeDirectory(fixturesDir, "i_do_not_exist") } } @Test - fun testOnExistingFile() { + fun `on existing file`() { assertThrows { assumeDirectory(fixturesDir, "simple_main.p8") } } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { val path = workingDir.div("..") assertThat( "should return resulting path", @@ -182,21 +182,21 @@ class PathsHelpersTests { @Nested inner class WithStringAndStringArgs { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { assertThrows { assumeDirectory("$fixturesDir", "i_do_not_exist") } } @Test - fun testOnExistingFile() { + fun `on existing file`() { assertThrows { assumeDirectory("$fixturesDir", "simple_main.p8") } } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { val path = workingDir.div("..") assertThat( "should return resulting path", @@ -208,21 +208,21 @@ class PathsHelpersTests { @Nested inner class WithStringAndPathArgs { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { assertThrows { assumeDirectory("$fixturesDir", Path("i_do_not_exist")) } } @Test - fun testOnExistingFile() { + fun `on existing file`() { assertThrows { assumeDirectory("$fixturesDir", Path("simple_main.p8")) } } @Test - fun testOnExistingDirectory() { + fun `on existing directory`() { val path = workingDir.div("..") assertThat( "should return resulting path", @@ -239,7 +239,7 @@ class PathsHelpersTests { inner class WithOnePathArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThrows { assumeReadableFile(path) @@ -247,14 +247,14 @@ class PathsHelpersTests { } @Test - fun testOnReadableFile() { + fun `on readable file`() { val path = fixturesDir.div("simple_main.p8") assertThat("should return the path", assumeReadableFile(path), `is`(path)) } @Test - fun testOnDirectory() { + fun `on directory`() { assertThrows { assumeReadableFile(fixturesDir) } @@ -265,7 +265,7 @@ class PathsHelpersTests { inner class WithOneStringArg { @Test - fun testOnNonExistingPath() { + fun `on non-existing path`() { val path = fixturesDir.div("i_do_not_exist") assertThrows { assumeReadableFile("$path") @@ -273,14 +273,14 @@ class PathsHelpersTests { } @Test - fun testOnReadableFile() { + fun `on readable file`() { val path = fixturesDir.div("simple_main.p8") assertThat("should return the resulting path", assumeReadableFile("$path"), `is`(path)) } @Test - fun testOnDirectory() { + fun `on directory`() { assertThrows { assumeReadableFile("$fixturesDir") } @@ -290,21 +290,21 @@ class PathsHelpersTests { @Nested inner class WithPathAndStringArgs { @Test - fun testOnNonexistingPath() { + fun `on non-existing path`() { assertThrows { assumeReadableFile(fixturesDir, "i_do_not_exist") } } @Test - fun testOnReadableFile() { + fun `on readable file`() { val path = fixturesDir.div("simple_main.p8") assertThat("should return the resulting path", assumeReadableFile(fixturesDir, "simple_main.p8"), `is`(path)) } @Test - fun testOnDirectory() { + fun `on directory`() { assertThrows { assumeReadableFile(fixturesDir, "..") } @@ -314,13 +314,13 @@ class PathsHelpersTests { @Nested inner class WithPathAndPathArgs { @Test - fun testOnNonexistingPath() { + fun `on non-existing path`() { assertThrows { assumeReadableFile(fixturesDir, Path("i_do_not_exist")) } } - @Test fun testOnReadableFile() { + @Test fun `on readable file`() { assertThat("should return the resulting path", assumeReadableFile(fixturesDir, Path("simple_main.p8")), `is`(fixturesDir.div("simple_main.p8")) @@ -328,7 +328,7 @@ class PathsHelpersTests { } @Test - fun testOnDirectory() { + fun `on directory`() { assertThrows { assumeReadableFile(fixturesDir, Path("..")) } @@ -338,14 +338,14 @@ class PathsHelpersTests { @Nested inner class WithStringAndStringArgs { @Test - fun testOnNonexistingPath() { + fun `on non-existing path`() { assertThrows { assumeReadableFile("$fixturesDir", "i_do_not_exist") } } @Test - fun testOnReadableFile() { + fun `on readable file`() { assertThat("should return the resulting path", assumeReadableFile(fixturesDir.toString(), "simple_main.p8"), `is`(fixturesDir.div("simple_main.p8")) @@ -353,7 +353,7 @@ class PathsHelpersTests { } @Test - fun testOnDirectory() { + fun `on directory`() { assertThrows { assumeReadableFile("$fixturesDir", "..") } From 2cbf2d22268f47d003490545213264a2c5e81b31 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Sep 2021 18:16:24 +0200 Subject: [PATCH 59/68] fix regression in imported module order (reversed) this caused an error in determining the main module and correct compilation options --- compiler/src/prog8/compiler/Compiler.kt | 4 +- .../TestImportedModulesOrderAndOptions.kt | 97 +++++++++++++++++++ compilerAst/src/prog8/ast/AstToplevel.kt | 2 +- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 compiler/test/TestImportedModulesOrderAndOptions.kt diff --git a/compiler/src/prog8/compiler/Compiler.kt b/compiler/src/prog8/compiler/Compiler.kt index e0ae421e1..e3166773e 100644 --- a/compiler/src/prog8/compiler/Compiler.kt +++ b/compiler/src/prog8/compiler/Compiler.kt @@ -168,7 +168,7 @@ private class BuiltinFunctionsFacade(functions: Map): IBuilt builtinFunctionReturnType(name, args, program) } -private fun parseImports(filepath: Path, +fun parseImports(filepath: Path, errors: IErrorReporter, compTarget: ICompilationTarget, libdirs: List): Triple> { @@ -200,7 +200,7 @@ private fun parseImports(filepath: Path, return Triple(programAst, compilerOptions, importedFiles) } -private fun determineCompilationOptions(program: Program, compTarget: ICompilationTarget): CompilationOptions { +fun determineCompilationOptions(program: Program, compTarget: ICompilationTarget): CompilationOptions { val mainModule = program.mainModule val outputDirective = (mainModule.statements.singleOrNull { it is Directive && it.directive == "%output" } as? Directive) val launcherDirective = (mainModule.statements.singleOrNull { it is Directive && it.directive == "%launcher" } as? Directive) diff --git a/compiler/test/TestImportedModulesOrderAndOptions.kt b/compiler/test/TestImportedModulesOrderAndOptions.kt new file mode 100644 index 000000000..7cb3d061b --- /dev/null +++ b/compiler/test/TestImportedModulesOrderAndOptions.kt @@ -0,0 +1,97 @@ +package prog8tests + +import org.junit.jupiter.api.* +import prog8.ast.internedStringsModuleName +import prog8.compiler.* +import prog8tests.helpers.* + +import prog8.compiler.target.C64Target +import kotlin.test.assertEquals +import kotlin.test.assertTrue + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestImportedModulesOrderAndOptions { + + @Test + fun testImportedModuleOrderCorrect() { + val result = compileText(C64Target, false, """ +%import textio +%import floats + +main { + sub start() { + ; nothing + } +} +""").assertSuccess() + assertTrue(result.programAst.mainModule.name.startsWith("on_the_fly_test")) + + val moduleNames = result.programAst.modules.map { it.name } + assertTrue(moduleNames[0].startsWith("on_the_fly_test"), "main module must be first") + assertEquals(listOf( + "prog8_interned_strings", + "textio", + "syslib", + "conv", + "floats", + "math", + "prog8_lib" + ), moduleNames.drop(1), "module order in parse tree") + } + + @Test + fun testCompilationOptionsCorrectFromMain() { + val result = compileText(C64Target, false, """ +%import textio +%import floats +%zeropage dontuse +%option no_sysinit + +main { + sub start() { + ; nothing + } +} +""").assertSuccess() + assertTrue(result.programAst.mainModule.name.startsWith("on_the_fly_test")) + val options = determineCompilationOptions(result.programAst, C64Target) + assertTrue(options.floats) + assertEquals(ZeropageType.DONTUSE, options.zeropage) + assertTrue(options.noSysInit) + } + + @Test + fun testModuleOrderAndCompilationOptionsCorrectWithJustImports() { + val errors = ErrorReporter() + val sourceText = """ +%import textio +%import floats +%option no_sysinit +%zeropage dontuse + +main { + sub start() { + ; nothing + } +} +""" + val filenameBase = "on_the_fly_test_" + sourceText.hashCode().toUInt().toString(16) + val filepath = outputDir.resolve("$filenameBase.p8") + filepath.toFile().writeText(sourceText) + val (program, options, importedfiles) = parseImports(filepath, errors, C64Target, emptyList()) + + assertEquals(filenameBase, program.mainModule.name) + assertEquals(1, importedfiles.size, "all imports other than the test source must have been internal resources library files") + assertEquals(listOf( + internedStringsModuleName, + filenameBase, + "textio", "syslib", "conv", "floats", "math", "prog8_lib" + ), program.modules.map {it.name}, "module order in parse tree") + assertTrue(options.floats) + assertEquals(ZeropageType.DONTUSE, options.zeropage, "zeropage option must be correctly taken from main module, not from float module import logic") + assertTrue(options.noSysInit) + } + + +} diff --git a/compilerAst/src/prog8/ast/AstToplevel.kt b/compilerAst/src/prog8/ast/AstToplevel.kt index 861f80f68..6d20882a5 100644 --- a/compilerAst/src/prog8/ast/AstToplevel.kt +++ b/compilerAst/src/prog8/ast/AstToplevel.kt @@ -251,7 +251,7 @@ class Program(val name: String, fun addModule(module: Module): Program { require(null == _modules.firstOrNull { it.name == module.name }) { "module '${module.name}' already present" } - _modules.add(0, module) + _modules.add(module) module.linkParents(namespace) module.program = this return this From 51452964862ebfd85d50ab2ea4539b1033b16b14 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Sep 2021 18:53:12 +0200 Subject: [PATCH 60/68] fix test assertion for float ranges (and re-enable test) --- compiler/test/TestCompilerOnRanges.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index 5f5fafdcb..904f44012 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -4,7 +4,6 @@ import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestFactory import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.DynamicTest.dynamicTest import kotlin.test.* import prog8tests.helpers.* @@ -105,7 +104,6 @@ class TestCompilerOnRanges { @TestFactory - @Disabled("#55") fun floatArrayInitializerWithRange() = mapCombinations( dim1 = listOf("", "42", "41"), // sizeInDecl dim2 = listOf("%option enable_floats", ""), // optEnableFloats @@ -113,6 +111,7 @@ class TestCompilerOnRanges { dim4 = listOf(false, true), // optimize combine4 = { sizeInDecl, optEnableFloats, platform, optimize -> val displayName = + "test failed for: " + when (sizeInDecl) { "" -> "no" "42" -> "correct" @@ -130,11 +129,10 @@ class TestCompilerOnRanges { } } """) - if ((sizeInDecl == "42") && (optEnableFloats != "")){ + if (optEnableFloats != "" && (sizeInDecl=="" || sizeInDecl=="42")) result.assertSuccess() - } else { + else result.assertFailure() - } } } ) From 7241cef7a5d9c8ae5be3125a70503ae9c718684f Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Sep 2021 18:59:53 +0200 Subject: [PATCH 61/68] fix char range in float-range test and exclude test.p8 example from tests --- compiler/test/TestCompilerOnExamples.kt | 1 - compiler/test/TestCompilerOnRanges.kt | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/test/TestCompilerOnExamples.kt b/compiler/test/TestCompilerOnExamples.kt index 0caa4d08c..2fe47b74e 100644 --- a/compiler/test/TestCompilerOnExamples.kt +++ b/compiler/test/TestCompilerOnExamples.kt @@ -75,7 +75,6 @@ class TestCompilerOnExamples { "swirl", "swirl-float", "tehtriz", - "test", "textelite", ), dim2 = listOf(Cx16Target, C64Target), diff --git a/compiler/test/TestCompilerOnRanges.kt b/compiler/test/TestCompilerOnRanges.kt index 904f44012..6bc3501b1 100644 --- a/compiler/test/TestCompilerOnRanges.kt +++ b/compiler/test/TestCompilerOnRanges.kt @@ -56,14 +56,13 @@ class TestCompilerOnRanges { } @Test - @Disabled("#55: bug in ConstantIdentifierReplacer.before(VarDecl)@decl.datatype==ARRAY_F") fun testFloatArrayInitializerWithRange_char_to_char() { val platform = C64Target val result = compileText(platform, optimize = false, """ %option enable_floats main { sub start() { - float[] cs = @'a' to 'z' ; values are computed at compile time + float[] cs = 'a' to 'z' ; values are computed at compile time cs[0] = 23 ; keep optimizer from removing it } } @@ -76,7 +75,7 @@ class TestCompilerOnRanges { val rhsValues = (decl.value as ArrayLiteralValue) .value // Array .map { (it as NumericLiteralValue).number.toInt() } - val expectedStart = platform.encodeString("a", true)[0].toInt() + val expectedStart = platform.encodeString("a", false)[0].toInt() val expectedEnd = platform.encodeString("z", false)[0].toInt() val expectedStr = "$expectedStart .. $expectedEnd" From 1a06e7a16e3f61234ed3be6bfd49360e58d00b30 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Sep 2021 19:02:07 +0200 Subject: [PATCH 62/68] expand range expression in float array decls, fixes issue #55 --- .../optimizer/ConstantIdentifierReplacer.kt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt index cfab08140..4efb5a6d5 100644 --- a/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt +++ b/compiler/src/prog8/optimizer/ConstantIdentifierReplacer.kt @@ -126,7 +126,6 @@ internal class ConstantIdentifierReplacer(private val program: Program, private } } DataType.ARRAY_UB, DataType.ARRAY_B, DataType.ARRAY_UW, DataType.ARRAY_W -> { - val numericLv = decl.value as? NumericLiteralValue val rangeExpr = decl.value as? RangeExpr if(rangeExpr!=null) { // convert the initializer range expression to an actual array @@ -148,6 +147,7 @@ internal class ConstantIdentifierReplacer(private val program: Program, private return listOf(IAstModification.ReplaceNode(decl.value!!, newValue, decl)) } } + val numericLv = decl.value as? NumericLiteralValue if(numericLv!=null && numericLv.type== DataType.FLOAT) errors.err("arraysize requires only integers here", numericLv.position) val size = decl.arraysize?.constIndex() ?: return noModifications @@ -180,14 +180,12 @@ internal class ConstantIdentifierReplacer(private val program: Program, private } } DataType.ARRAY_F -> { - val size = decl.arraysize?.constIndex() ?: return noModifications - val litval = decl.value as? NumericLiteralValue val rangeExpr = decl.value as? RangeExpr if(rangeExpr!=null) { // convert the initializer range expression to an actual array of floats val declArraySize = decl.arraysize?.constIndex() if(declArraySize!=null && declArraySize!=rangeExpr.size(compTarget)) - errors.err("range expression size doesn't match declared array size", decl.value?.position!!) + errors.err("range expression size (${rangeExpr.size(compTarget)}) doesn't match declared array size ($declArraySize)", decl.value?.position!!) val constRange = rangeExpr.toConstantIntegerRange(compTarget) if(constRange!=null) { val newValue = ArrayLiteralValue(InferredTypes.InferredType.known(DataType.ARRAY_F), @@ -196,15 +194,18 @@ internal class ConstantIdentifierReplacer(private val program: Program, private return listOf(IAstModification.ReplaceNode(decl.value!!, newValue, decl)) } } - if(rangeExpr==null && litval!=null) { + + val numericLv = decl.value as? NumericLiteralValue + val size = decl.arraysize?.constIndex() ?: return noModifications + if(rangeExpr==null && numericLv!=null) { // arraysize initializer is a single int, and we know the size. - val fillvalue = litval.number.toDouble() + val fillvalue = numericLv.number.toDouble() if (fillvalue < compTarget.machine.FLOAT_MAX_NEGATIVE || fillvalue > compTarget.machine.FLOAT_MAX_POSITIVE) - errors.err("float value overflow", litval.position) + errors.err("float value overflow", numericLv.position) else { // create the array itself, filled with the fillvalue. - val array = Array(size) {fillvalue}.map { NumericLiteralValue(DataType.FLOAT, it, litval.position) }.toTypedArray() - val refValue = ArrayLiteralValue(InferredTypes.InferredType.known(DataType.ARRAY_F), array, position = litval.position) + val array = Array(size) {fillvalue}.map { NumericLiteralValue(DataType.FLOAT, it, numericLv.position) }.toTypedArray() + val refValue = ArrayLiteralValue(InferredTypes.InferredType.known(DataType.ARRAY_F), array, position = numericLv.position) return listOf(IAstModification.ReplaceNode(decl.value!!, refValue, decl)) } } From 5988ba76b543e91ad493e55d3a9d87b148e81a29 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 12 Sep 2021 19:09:50 +0200 Subject: [PATCH 63/68] test example for fixed float ranges --- examples/test.p8 | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/examples/test.p8 b/examples/test.p8 index 860fab040..83541711f 100644 --- a/examples/test.p8 +++ b/examples/test.p8 @@ -1,42 +1,37 @@ %import textio +%import floats %zeropage dontuse main { label: sub start() { - sub2(&label) - sub2(&label_local) - sub2(&main.sub2.label_in_sub2) - uword xx = &label_local - txt.print_uwhex(xx, true) + + ubyte[6] ubs = 10 to 20 step 2 + ubyte[] ubs2 = 10 to 20 step 2 + float[6] fs = 10 to 20 step 2 + float[] fs2 = 10 to 20 step 2 + + txt.print_ub(len(ubs)) txt.nl() - xx = &label - txt.print_uwhex(xx, true) + txt.print_ub(len(ubs2)) txt.nl() - xx = &main.label - txt.print_uwhex(xx, true) + txt.print_ub(len(fs)) txt.nl() - xx = &main.sub2.label_in_sub2 - txt.print_uwhex(xx, true) - txt.nl() - xx = main.sub2.sub2var - txt.print_uwhex(xx, true) - txt.nl() - xx = &main.start.label_local - txt.print_uwhex(xx, true) + txt.print_ub(len(fs2)) txt.nl() -label_local: - return - } - - sub sub2(uword ad) { - uword sub2var = 42 - - txt.print_uwhex(ad,true) + ubyte ix + for ix in 0 to 5 { + txt.print_ub(ubs2[ix]) + txt.spc() + } txt.nl() -label_in_sub2: + for ix in 0 to 5 { + floats.print_f(fs2[ix]) + txt.spc() + } txt.nl() + } } From 2365a076ac3e6c74d1f6335365744fa09f03c0cc Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 9 Oct 2021 16:33:52 +0200 Subject: [PATCH 64/68] clean test.p8 --- examples/test.p8 | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/examples/test.p8 b/examples/test.p8 index 83541711f..03f7faf6e 100644 --- a/examples/test.p8 +++ b/examples/test.p8 @@ -1,37 +1,10 @@ %import textio -%import floats -%zeropage dontuse +%import test_stack +%zeropage basicsafe main { - -label: sub start() { - - ubyte[6] ubs = 10 to 20 step 2 - ubyte[] ubs2 = 10 to 20 step 2 - float[6] fs = 10 to 20 step 2 - float[] fs2 = 10 to 20 step 2 - - txt.print_ub(len(ubs)) - txt.nl() - txt.print_ub(len(ubs2)) - txt.nl() - txt.print_ub(len(fs)) - txt.nl() - txt.print_ub(len(fs2)) - txt.nl() - - ubyte ix - for ix in 0 to 5 { - txt.print_ub(ubs2[ix]) - txt.spc() - } - txt.nl() - for ix in 0 to 5 { - floats.print_f(fs2[ix]) - txt.spc() - } - txt.nl() - + txt.print("ok") + test_stack.test() } } From 562d8386ec1aefc699fcd782626a96f8a92270ba Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 9 Oct 2021 16:57:56 +0200 Subject: [PATCH 65/68] fix antlr generator settings --- .idea/misc.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 5455bfad2..d79c99176 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -4,7 +4,7 @@ - + - \ No newline at end of file + From 371d4768e6ebfaeed143aa70efde066a0fd3eb67 Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 9 Oct 2021 17:59:40 +0200 Subject: [PATCH 66/68] fix filename case issue --- compiler/test/TestCompilerOnImportsAndIncludes.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/test/TestCompilerOnImportsAndIncludes.kt b/compiler/test/TestCompilerOnImportsAndIncludes.kt index 904f0fd3d..f70c7eede 100644 --- a/compiler/test/TestCompilerOnImportsAndIncludes.kt +++ b/compiler/test/TestCompilerOnImportsAndIncludes.kt @@ -108,7 +108,7 @@ class TestCompilerOnImportsAndIncludes { inner class Asmbinary { @Test fun testAsmbinaryDirectiveWithNonExistingFile() { - val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonExisting.p8") + val p8Path = assumeReadableFile(fixturesDir, "asmBinaryNonExisting.p8") assumeNotExists(fixturesDir, "i_do_not_exist.bin") compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) @@ -117,7 +117,7 @@ class TestCompilerOnImportsAndIncludes { @Test fun testAsmbinaryDirectiveWithNonReadableFile() { - val p8Path = assumeReadableFile(fixturesDir, "asmbinaryNonReadable.p8") + val p8Path = assumeReadableFile(fixturesDir, "asmBinaryNonReadable.p8") assumeDirectory(fixturesDir, "subFolder") compileFile(Cx16Target, false, p8Path.parent, p8Path.name, outputDir) @@ -151,4 +151,4 @@ class TestCompilerOnImportsAndIncludes { } -} \ No newline at end of file +} From dbe98f3fa57e5321a1cac70ff66233e9d811040f Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sat, 9 Oct 2021 18:43:18 +0200 Subject: [PATCH 67/68] remove unittest of %target directive, which is removed in 7.1 --- compilerAst/test/TestAstToSourceCode.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/compilerAst/test/TestAstToSourceCode.kt b/compilerAst/test/TestAstToSourceCode.kt index 4a27c7463..b179b48b9 100644 --- a/compilerAst/test/TestAstToSourceCode.kt +++ b/compilerAst/test/TestAstToSourceCode.kt @@ -47,14 +47,6 @@ class TestAstToSourceCode { assertContains(txt, Regex(";.*$internedStringsModuleName")) } - @Test - fun testTargetDirectiveAndComment() { - val orig = SourceCode.of("%target 42 ; invalid arg - shouldn't matter\n") - val (txt, _) = roundTrip(parseModule(orig)) - // assertContains has *actual* first! - assertContains(txt, Regex("%target +42")) - } - @Test fun testImportDirectiveWithLib() { val orig = SourceCode.of("%import textio\n") From f4b3d1905979650a1e6dceb45e516711c433bd2c Mon Sep 17 00:00:00 2001 From: Irmen de Jong Date: Sun, 10 Oct 2021 22:26:18 +0200 Subject: [PATCH 68/68] fix merge conflict --- compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index 7510d048d..f1e991129 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -12,13 +12,6 @@ import prog8.parser.Prog8ANTLRParser private data class NumericLiteral(val number: Number, val datatype: DataType) -// TODO [merge conflict]: not sure if this should be kept?? Is it double?? -internal fun Prog8ANTLRParser.ModuleContext.toAst(name: String, source: Path, encoding: IStringEncoding) : Module { - val nameWithoutSuffix = if(name.endsWith(".p8")) name.substringBeforeLast('.') else name - val directives = this.directive().map { it.toAst() } - val blocks = this.block().map { it.toAst(Module.isLibrary(source), encoding) } - return Module(nameWithoutSuffix, (directives + blocks).toMutableList(), toPosition(), source) -} private fun ParserRuleContext.toPosition() : Position { /*