From 2540b28ca3b2b5e4649d9d5cc57c5061755bf4af Mon Sep 17 00:00:00 2001
From: Stephen Heumann <stephenheumann@gmail.com>
Date: Wed, 15 Feb 2023 22:04:10 -0600
Subject: [PATCH] Avoid excessively deep recursion in qsort.

It could have O(n) recursion depth for some inputs (e.g. if already sorted or reverse sorted), which could easily cause stack overflows.

Now, recursion is only used for the smaller of the two subarrays at each step, so the maximum recursion depth is bounded to log2(n).
---
 stdlib.asm    | 26 ++++++++++++++++++--------
 stdlib.macros | 12 ++++++++++++
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/stdlib.asm b/stdlib.asm
index 329c67b..20f63f5 100644
--- a/stdlib.asm
+++ b/stdlib.asm
@@ -767,10 +767,10 @@ right    equ   5                        right address
 
          csubroutine (4:first,4:last),8
 
-         phb
+sr0      phb
          phk
          plb
-sr0      lda   last+2                   if last <= first then quit
+         lda   last+2                   if last <= first then quit
          cmp   first+2
          bne   sr1
          lda   last
@@ -825,15 +825,24 @@ sr5      blt   sr2
          ph4   <left                    swap left/last entries
          ph4   <last
          jsr   swap
-         sub4  left,lsize,right         sort left part of array
+         sub4  left,lsize,right         calculate bounds of subarrays
+         add4  left,lsize                 (first..right and left..last)
+         add4  first,last,mid           calculate midpoint of range being sorted
+         lsr   mid+2
+         ror   mid
+         cmpl  right,mid                if right < mid then
+         bge   sr6
          plb
-         ph4   <right
+         ph4   <right                     sort left subarray recursively
          ph4   <first
          jsl   rsort
-         phb
-         phk
-         plb
-         add4  left,lsize,first         sort right part of array
+         move4 left,first                 sort right subarray via tail call
+         brl   sr0
+sr6      plb                            else
+         ph4   <last                      sort right subarray recursively
+         ph4   <left
+         jsl   rsort
+         move4 right,last                 sort left subarray via tail call
          brl   sr0
 ;
 ;  swap - swap two entries
@@ -909,6 +918,7 @@ sw6      pld
 lsize    entry
          ds    4                        local copy of size
 banks    ds    2                        number of whole banks to swap
+mid      ds    4                        midpoint of the elements being sorted
          end
 
 ****************************************************************
diff --git a/stdlib.macros b/stdlib.macros
index 449ec86..a0d378a 100644
--- a/stdlib.macros
+++ b/stdlib.macros
@@ -734,3 +734,15 @@
 &l bne *+5
  brl &bp
  mend
+ macro
+&l cmpl &n1,&n2
+ lclb &yistwo
+&l ~setm
+ ~lda.h &n1
+ ~op.h cmp,&n2
+ bne ~a&SYSCNT
+ ~lda &n1
+ ~op cmp,&n2
+~a&SYSCNT anop
+ ~restm
+ mend