From 7384c82667ee46282bc17968b2c57076dea7118c Mon Sep 17 00:00:00 2001
From: Stephen Heumann <stephenheumann@gmail.com>
Date: Tue, 20 Feb 2024 22:20:17 -0600
Subject: [PATCH] fclose: Check for malloc failure when closing temp files.

Previously, the code for closing a temporary file assumed that malloc would succeed. If it did not, the code would trash memory and (at least in my testing) crash the system. Now it checks for and handles malloc failures, although they will still lead to the temporary file not being deleted.

Here is a test program illustrating the problem:

#include <stdio.h>
#include <stdlib.h>

int main(void) {
        FILE *f = tmpfile();
        if (!f)
                return 0;

        void *p;
        do {
                p = malloc(8*1024);
        } while (p);

        fclose(f);
}
---
 stdio.asm | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/stdio.asm b/stdio.asm
index 64a7b54..d2f5a79 100644
--- a/stdio.asm
+++ b/stdio.asm
@@ -129,12 +129,18 @@ cl3a     ldy   #FILE_flag               if the file was opened by tmpfile then
          and   #_IOTEMPFILE
          beq   cl3d
          ph4   #nameBuffSize              p = malloc(nameBuffSize)
-         jsl   malloc                     grPathname = p
-         sta   p                          dsPathname = p+2
+         jsl   malloc
+         sta   p
          stx   p+2
-         sta   grPathname
+         ora   p+2                        if p == NULL then
+         bne   cl3aa
+         lda   #EOF                         flag error
+         sta   err
+         bra   cl3d                         just close the file
+cl3aa    lda   p
+         sta   grPathname                 grPathname = p
          stx   grPathname+2
-         clc
+         clc                              dsPathname = p+2
          adc   #2
          bcc   cl3b
          inx