Merge pull request #118 from greg-king5/quoted-token

Add a quoted-token version of strtok().

doc/funcref.sgml

@@ -3,7 +3,7 @@
 <article>
 <title>cc65 function reference
 <author><url url="mailto:uz@cc65.org" name="Ullrich von Bassewitz">
-<date>2014-04-24
+<date>2014-05-22
 
 <abstract>
 cc65 is a C compiler for 6502 based systems. This function reference describes
@@ -608,6 +608,7 @@ communication.
 <item><ref id="strncat" name="strncat">
 <item><ref id="strncmp" name="strncmp">
 <item><ref id="strncpy" name="strncpy">
+<item><ref id="strqtok" name="strqtok">
 <item><ref id="strrchr" name="strrchr">
 <item><ref id="strspn" name="strspn">
 <item><ref id="strstr" name="strstr">
@@ -5733,6 +5734,7 @@ be used in presence of a prototype.
 </itemize>
 <tag/Availability/ISO 9899
 <tag/See also/
+<ref id="strqtok" name="strqtok">,
 <ref id="strspn" name="strspn">,
 <ref id="strstr" name="strstr">,
 <ref id="strtok" name="strtok">
@@ -5965,6 +5967,38 @@ hello[5] = '\0';
 </quote>
 
 
+<sect1>strqtok<label id="strqtok"><p>
+
+<quote>
+<descrip>
+<tag/Function/Break a string into tokens.
+<tag/Header/<tt/<ref id="string.h" name="string.h">/
+<tag/Declaration/<tt/char* __fastcall__ strqtok (char* s1, const char* s2);/
+<tag/Description/<tt/strqtok()/ will break the string <tt/s1/ into a sequence of
+tokens, which are delimited by either quotation marks or characters from the
+string <tt/s2/. Tokens inside quotation marks may contain characters from <tt/s2/
+(they aren't delimiters there). The first call to <tt/strqtok()/ will return a
+pointer to the first token in the string <tt/s1/. The following calls must pass
+a <tt/NULL/ pointer as <tt/s1/, in order to get the next token in the string.
+Different sets of delimiters may be used for the subsequent calls to <tt/strqtok()/.
+<tag/Limits/<itemize>
+<item>The function is available only as a fastcall function; so, it may be used
+only in the presence of a prototype.
+<item><tt/strqtok()/ will modify the string <tt/s1/.
+<item>The function will forget where it is in the <tt/s1/ string if it is given
+a second <tt/s1/ string before it finishes the first one.
+</itemize>
+<tag/Availability/cc65
+<tag/See also/
+<ref id="strcspn" name="strcspn">,
+<!-- <ref id="strpbrk" name="strpbrk">, -->
+<ref id="strspn" name="strspn">,
+<ref id="strtok" name="strtok">
+<tag/Example/None.
+</descrip>
+</quote>
+
+
 <sect1>strrchr<label id="strrchr"><p>
 
 <quote>
@@ -6041,21 +6075,25 @@ be used in presence of a prototype.
 <tag/Function/Break a string into tokens.
 <tag/Header/<tt/<ref id="string.h" name="string.h">/
 <tag/Declaration/<tt/char* __fastcall__ strtok (char* s1, const char* s2);/
-<tag/Description/<tt/strtok/ will break the string s1 into a sequence of
+<tag/Description/<tt/strtok()/ will break the string <tt/s1/ into a sequence of
-tokens, which are delimited by characters from the string s2. The first call
+tokens, which are delimited by characters from the string <tt/s2/. The first call
-to <tt/strtok/ will return a pointer to the first token in the string s1.
+to <tt/strtok()/ will return a pointer to the first token in the string <tt/s1/.
-Following calls must pass a <tt/NULL/ pointer as s1 in order to get the next
+The following calls must pass a <tt/NULL/ pointer as <tt/s1/, in order to get
-token in the string. Different sets of delimiters may be used for the
+the next token in the string. Different sets of delimiters may be used for the
-subsequent calls to <tt/strtok/.
+subsequent calls to <tt/strtok()/.
 <tag/Limits/<itemize>
 <item>The function is only available as fastcall function, so it may only
 be used in presence of a prototype.
-<item><tt/strtok/ will modify the string s1.
+<item><tt/strtok()/ will modify the string <tt/s1/.
+<item>The function will forget where it is in the <tt/s1/ string if it is given
+a second <tt/s1/ string before it finishes the first one.
 </itemize>
 <tag/Availability/ISO 9899
 <tag/See also/
 <ref id="strcspn" name="strcspn">,
-<!-- <ref id="strpbrk" name="strpbrk"> -->
+<!-- <ref id="strpbrk" name="strpbrk">, -->
+<ref id="strqtok" name="strqtok">,
+<ref id="strspn" name="strspn">
 <tag/Example/None.
 </descrip>
 </quote>

include/string.h

@@ -6,10 +6,10 @@
 /*                                                                           */
 /*                                                                           */
 /*                                                                           */
-/* (C) 1998-2008 Ullrich von Bassewitz                                       */
+/* (C) 1998-2014, Ullrich von Bassewitz                                      */
-/*               Roemerstrasse 52                                            */
+/*                Roemerstrasse 52                                           */
-/*               D-70794 Filderstadt                                         */
+/*                D-70794 Filderstadt                                        */
-/* EMail:        uz@cc65.org                                                 */
+/* EMail:         uz@cc65.org                                                */
 /*                                                                           */
 /*                                                                           */
 /* This software is provided 'as-is', without any expressed or implied       */
@@ -81,6 +81,7 @@ char* __fastcall__ strlwr (char* s);
 char* __fastcall__ strlower (char* s);
 char* __fastcall__ strupr (char* s);
 char* __fastcall__ strupper (char* s);
+char* __fastcall__ strqtok (char* s1, const char* s2);
 #endif
 
 const char* __fastcall__ _stroserror (unsigned char errcode);
@@ -90,6 +91,3 @@ const char* __fastcall__ _stroserror (unsigned char errcode);
 
 /* End of string.h */
 #endif
-
-
-

libsrc/common/strqtok.c

@@ -0,0 +1,86 @@
+/*
+ * strqtok() is like strtok():  It finds pieces of text, in a string, that are
+ * surrounded by given delimiter characters.  It returns each piece, in turn,
+ * as a string, until every piece has been found.  Then, it returns NULL.  But,
+ * strqtok() recognizes quotation marks.  A mark makes delimiters look ordinary
+ * until another quotation mark is seen.  That allows us to include delimiters
+ * in tokens.  (This version doesn't allow escaped quotation marks.)
+ *
+ * 2014-04-19, Daniel Serpell
+ * 2014-04-21, Paul Foerster
+ * 2014-04-25, Greg King
+ */
+
+
+#include <string.h>
+
+
+char* __fastcall__ strqtok (register char* s1, const char* s2)
+{
+    static char  c;
+    static char* start;
+    static char* next = "";
+
+    if (s1 == NULL) {
+        s1 = next;
+        if (c == '\"') {
+            goto inQuote;
+        }
+    }
+
+    /* Search for the start of a token. */
+    while (strchr (s2, c = *s1)) {
+        if (c == '\0') {
+            /* No more tokens. */
+            return NULL;
+        }
+        ++s1;
+    }
+    if (c == '\"') {
+        goto skipQuote;
+    }
+
+    /* Save the start of the token. */
+    start = s1;
+
+    /* Search for the end of a non-quoted token. */
+    while ((c = *s1) != '\"' && !strchr (s2, c)) {
+        ++s1;
+    }
+    if (c == '\0') {
+        /* The end of the last token is the end of the token list;
+         * don't go beyond it.
+         */
+        goto found;
+    }
+
+    /* (A possible begin-quote mark will be rememberred.) */
+    goto terminate;
+
+  skipQuote:
+    ++s1;
+
+  inQuote:
+    /* Don't let a quote mark be rememberred. */
+    c = '\0';
+
+    /* Save the start of the token. */
+    start = s1;
+
+    /* Search for the end of a quoted token. */
+    if ((s1 = strchr (s1, '\"')) == NULL) {
+        /* The quoted token ended with '\0'; therefore, point to a '\0',
+         * so that the next call will return NULL.
+         */
+        next = "";
+        return start;
+    }
+
+  terminate:
+    *s1 = '\0';
+    ++s1;
+
+  found:
+    next = s1;
+    return start;
+}

testcode/lib/strqtok-test.c

@@ -0,0 +1,51 @@
+/* strqtok-test.c
+ *
+ * 2014-04-21, Paul Foerster
+ * 2014-05-20, Greg King
+ *
+ * This program tests that strqtok() correctly will parse strings
+ * with quotation marks in them.  It should show this list of tokens
+ * from the test strings:
+ *
+ * >This<
+ * >  is only    <
+ * >a<
+ * >short<
+ * >quoting<
+ * >test ,  honoring  blanks, commas<
+ * >and<
+ * >(4)<
+ * >empty<
+ * ><
+ * ><
+ * ><
+ * ><
+ * >strings,   EOT  <
+ *
+ * It shouldn't show
+ *
+ * >Bogus token<
+ *
+ */
+
+#include <string.h>
+#include <stdio.h>
+
+void main (void)
+{
+    /* b[] and s[] are declared as automatic, not static, variables
+     * because strqtok() will change them.
+     * They must be defined together; and, b[] must be defined first
+     * (because they're copied onto the top-down stack).
+     */
+    char  b[] = "Bogus token ";
+    char  s[] = "  This ,  \"  is only    \"a   short   "
+        "quoting\"test ,  honoring  blanks"
+        ", commas\", and (4) empty \"\"\"\"\"\"\"\" \"strings,   EOT  ";
+    char* t = strqtok (s, " ,");
+
+    while (t != NULL) {
+        printf (">%s<\n", t);
+        t = strqtok (NULL, " ,");
+    }
+}