From b3f028da2feb5d4a38dbcfa70ac7c2b587eb29b9 Mon Sep 17 00:00:00 2001
From: Stephen Heumann <stephenheumann@gmail.com>
Date: Thu, 16 Feb 2023 18:44:47 -0600
Subject: [PATCH] Avoid address comparison error in qsort.

If the last element in the range being sorted has the smallest value, rsort can be called with last set to first-1, i.e. pointing to (what would be) the element before the first one. But with large enough element sizes and appropriate address values, this address computation can wrap around and produce a negative value for last. We need to treat such a value as being less than first, so it terminates that branch of the recursive computation. Previously, we were doing an unsigned comparison, so such a last value would be treated as greater than first and would lead to improper behavior including memory trashing.

Here is an example program that can show this (depending on memory layout):

#pragma memorymodel 1
#include <stdlib.h>
#include <stdio.h>

#define PADSIZE 2000000 /* may need to adjust based on memory size/layout */
#define N 2

struct big {
        int i;
        char pad[PADSIZE];
};

int cmp(const void *p1, const void *p2) {
        int a = ((struct big *)p1)->i;
        int b = ((struct big *)p2)->i;

        return (a < b) ? -1 : (a > b);
}

int main(void) {
        int j;
        struct big *p = malloc(sizeof(struct big) * N);
        if (!p)
                return 0;

        for (j = 0; j < N; j++) {
                p[j].i = N-j;
        }

        qsort(p, N, sizeof(struct big), cmp);

        for (j = 0; j < N; j++) {
                printf("%i\n", p[j].i);
        }
}
---
 stdlib.asm | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/stdlib.asm b/stdlib.asm
index 20f63f5..0c30719 100644
--- a/stdlib.asm
+++ b/stdlib.asm
@@ -771,15 +771,16 @@ sr0      phb
          phk
          plb
          lda   last+2                   if last <= first then quit
+         bmi   sr1a
          cmp   first+2
          bne   sr1
          lda   last
          cmp   first
-sr1      bgt   sr1a
-         plb
+sr1      bgt   sr1b
+sr1a     plb
          creturn
 
-sr1a     move4 last,right               right = last
+sr1b     move4 last,right               right = last
          move4 first,left               left = first
          bra   sr3
 sr2      add4  left,lsize               inc left until *left >= *last