Added support for precalculated checksum and (pregenerated) HTTP/1.1 headers ("connection: close" and "content-length")

apps/httpserver_raw/fs.c

@@ -29,6 +29,7 @@
  * Author: Adam Dunkels <adam@sics.se>
  *
  */
+#include "lwip/opt.h"
 #include "lwip/def.h"
 #include "fs.h"
 #include "fsdata.h"
@@ -44,7 +45,7 @@
 /* Define the file system memory allocation structure. */
 struct fs_table {
   struct fs_file file;
-  int inuse;
+  u8_t inuse;
 };
 
 /* Allocate file system memory */
@@ -97,6 +98,10 @@ fs_open(const char *name)
       file->index = f->len;
       file->pextension = NULL;
       file->http_header_included = f->http_header_included;
+#if HTTPD_PRECALCULATED_CHECKSUM
+      file->chksum_count = f->chksum_count;
+      file->chksum = f->chksum;
+#endif /* HTTPD_PRECALCULATED_CHECKSUM */
       return file;
     }
   }
@@ -125,7 +130,7 @@ fs_read(struct fs_file *file, char *buffer, int count)
     read = count;
   }
 
-  memcpy(buffer, (file->data + file->index), read);
+  MEMCPY(buffer, (file->data + file->index), read);
   file->index += read;
 
   return(read);

apps/httpserver_raw/fs.h

@@ -32,17 +32,36 @@
 #ifndef __FS_H__
 #define __FS_H__
 
+#include "lwip/opt.h"
+
+/** HTTPD_PRECALCULATED_CHECKSUM==1: include precompiled checksums for
+ * predefined (MSS-sized) chunks of the files to prevent having to calculate
+ * the checksums at runtime. */
+#ifndef HTTPD_PRECALCULATED_CHECKSUM
+#define HTTPD_PRECALCULATED_CHECKSUM 0
+#endif
+
+#if HTTPD_PRECALCULATED_CHECKSUM
+struct fsdata_chksum {
+  u32_t offset;
+  u16_t chksum;
+  u16_t len;
+};
+#endif /* HTTPD_PRECALCULATED_CHECKSUM */
+
 struct fs_file {
   const char *data;
   int len;
   int index;
   void *pextension;
-  int http_header_included;
+#if HTTPD_PRECALCULATED_CHECKSUM
+  const struct fsdata_chksum *chksum;
+  u16_t chksum_count;
+#endif /* HTTPD_PRECALCULATED_CHECKSUM */
+  u8_t http_header_included;
 };
 
-/* file must be allocated by caller and will be filled in
+struct fs_file *fs_open(const char *name);
-   by the function. */
-struct fs_file * fs_open(const char *name);
 void fs_close(struct fs_file *file);
 int fs_read(struct fs_file *file, char *buffer, int count);
 int fs_bytes_left(struct fs_file *file);

apps/httpserver_raw/fsdata.c

@@ -263,21 +263,24 @@ file_NULL,
 data__img_sics_gif,
 data__img_sics_gif + 16,
 sizeof(data__img_sics_gif) - 16,
-1}};
+1,
+}};
 
 const struct fsdata_file file__404_html[] = { {
 file__img_sics_gif,
 data__404_html,
 data__404_html + 12,
 sizeof(data__404_html) - 12,
-1}};
+1,
+}};
 
 const struct fsdata_file file__index_html[] = { {
 file__404_html,
 data__index_html,
 data__index_html + 12,
 sizeof(data__index_html) - 12,
-1}};
+1,
+}};
 
 #define FS_ROOT file__index_html
 #define FS_NUMFILES 3

apps/httpserver_raw/fsdata.h

@@ -32,12 +32,19 @@
 #ifndef __FSDATA_H__
 #define __FSDATA_H__
 
+#include "lwip/opt.h"
+#include "fs.h"
+
 struct fsdata_file {
   const struct fsdata_file *next;
   const unsigned char *name;
   const unsigned char *data;
   int len;
-  int http_header_included;
+  u8_t http_header_included;
+#if HTTPD_PRECALCULATED_CHECKSUM
+  u16_t chksum_count;
+  const struct fsdata_chksum *chksum;
+#endif /* HTTPD_PRECALCULATED_CHECKSUM */
 };
 
 #endif /* __FSDATA_H__ */

apps/httpserver_raw/makefsdata/makefsdata.c

@@ -6,6 +6,9 @@
  * Author: Jim Pettinato
  *         Simon Goldschmidt
  *
+ * @todo:
+ * - take TCP_MSS, LWIP_TCP_TIMESTAMPS and
+ *   PAYLOAD_ALIGN_TYPE/PAYLOAD_ALIGNMENT as arguments
  */
 
 #include <stdio.h>
@@ -60,6 +63,9 @@
 #define LWIP_HTTPD_DYNAMIC_HEADERS 1
 #include "../httpd_structs.h"
 
+#include "../../../../lwip/src/core/ipv4/inet_chksum.c"
+#include "../../../../lwip/src/core/def.c"
+
 /** (Your server name here) */
 const char *serverID = "Server: "HTTPD_SERVER_AGENT"\r\n";
 
@@ -75,12 +81,14 @@ static int payload_alingment_dummy_counter = 0;
 
 #define MAX_PATH_LEN 256
 
-#define COPY_BUFSIZE 1024
+#define COPY_BUFSIZE 10240
 
 int process_sub(FILE *data_file, FILE *struct_file);
 int process_file(FILE *data_file, FILE *struct_file, const char *filename);
-int file_write_http_header(FILE *data_file, const char *filename, int file_size);
+int file_write_http_header(FILE *data_file, const char *filename, int file_size,
+                           u16_t *http_hdr_len, u16_t *http_hdr_chksum);
 int file_put_ascii(FILE *file, const char *ascii_string, int len, int *i);
+int s_put_ascii(char *buf, const char *ascii_string, int len, int *i);
 void concat_files(const char *file1, const char *file2, const char *targetfile);
 
 static unsigned char file_buffer_raw[COPY_BUFSIZE];
@@ -89,10 +97,12 @@ static char file_buffer_c[COPY_BUFSIZE * 5 + ((COPY_BUFSIZE / HEX_BYTES_PER_LINE
 
 char curSubdir[MAX_PATH_LEN];
 char lastFileVar[MAX_PATH_LEN];
+char hdr_buf[4096];
 
 unsigned char processSubs = 1;
 unsigned char includeHttpHeader = 1;
 unsigned char useHttp11 = 0;
+unsigned char precalcChksum = 0;
 
 int main(int argc, char *argv[])
 {
@@ -120,6 +130,8 @@ int main(int argc, char *argv[])
       includeHttpHeader = 0;
     } else if (strstr(argv[i], "-11")) {
       useHttp11 = 1;
+    } else if (strstr(argv[i], "-c")) {
+      precalcChksum = 1;
     } else {
       strcpy(path, argv[i]);
     }
@@ -133,8 +145,9 @@ int main(int argc, char *argv[])
     printf(" Usage: htmlgen [targetdir] [-s] [-i]" NEWLINE NEWLINE);
     printf("   targetdir: relative or absolute path to files to convert" NEWLINE);
     printf("   switch -s: toggle processing of subdirectories (default is on)" NEWLINE);
-    printf("   switch -e: exclude HTTP header from file (header is created at runtime, default is on)" NEWLINE);
+    printf("   switch -e: exclude HTTP header from file (header is created at runtime, default is off)" NEWLINE);
     printf("   switch -11: include HTTP 1.1 header (1.0 is default)" NEWLINE);
+    printf("   switch -c: precalculate checksums for all pages (default is off)" NEWLINE);
     printf("   if targetdir not specified, htmlgen will attempt to" NEWLINE);
     printf("   process files in subdirectory 'fs'" NEWLINE);
     exit(-1);
@@ -154,7 +167,15 @@ int main(int argc, char *argv[])
 
   GETCWD(appPath, MAX_PATH_LEN);
   data_file = fopen("fsdata.tmp", "wb");
+  if (data_file == NULL) {
+    printf("Failed to create file \"fsdata.tmp\"\n");
+    exit(-1);
+  }
   struct_file = fopen("fshdr.tmp", "wb");
+  if (struct_file == NULL) {
+    printf("Failed to create file \"fshdr.tmp\"\n");
+    exit(-1);
+  }
 
   CHDIR(path);
 
@@ -182,6 +203,10 @@ int main(int argc, char *argv[])
   printf(NEWLINE "Creating target file..." NEWLINE NEWLINE);
   concat_files("fsdata.tmp", "fshdr.tmp", "fsdata.c");
 
+  /* if succeeded, delete the temporary files */
+  remove("fsdata.tmp");
+  remove("fshdr.tmp"); 
+
   printf(NEWLINE "Processed %d files - done." NEWLINE NEWLINE, filesProcessed);
 
   return 0;
@@ -191,7 +216,11 @@ static void copy_file(const char *filename_in, FILE *fout)
 {
   FILE *fin;
   size_t len;
-  fin = fopen(filename_in, "r");
+  fin = fopen(filename_in, "rb");
+  if (fin == NULL) {
+    printf("Failed to open file \"%s\"\n", filename_in);
+    exit(-1);
+  }
 
   while((len = fread(file_buffer_raw, 1, COPY_BUFSIZE, fin)) > 0)
   {
@@ -204,6 +233,10 @@ void concat_files(const char *file1, const char *file2, const char *targetfile)
 {
   FILE *fout;
   fout = fopen(targetfile, "wb");
+  if (fout == NULL) {
+    printf("Failed to open file \"%s\"\n", targetfile);
+    exit(-1);
+  }
   copy_file(file1, fout);
   copy_file(file2, fout);
   fclose(fout);
@@ -261,11 +294,13 @@ int get_file_size(const char* filename)
   FILE *inFile;
   int file_size = -1;
   inFile = fopen(filename, "rb");
-  if(inFile) {
+  if (inFile == NULL) {
-    fseek (inFile , 0 , SEEK_END);
+    printf("Failed to open file \"%s\"\n", filename);
-    file_size = ftell(inFile);
+    exit(-1);
-    fclose(inFile);
   }
+  fseek(inFile, 0, SEEK_END);
+  file_size = ftell(inFile);
+  fclose(inFile);
   return file_size;
 }
 
@@ -290,7 +325,52 @@ void process_file_data(const char *filename, FILE *data_file)
       }
       written = fwrite(file_buffer_c, 1, off, data_file);
     }
-  }while(len > 0);
+  } while(len > 0);
+  fclose(source_file);
+}
+
+int write_checksums(FILE *struct_file, const char *filename, const char *varname,
+                    u16_t hdr_len, u16_t hdr_chksum)
+{
+  int chunk_size = TCP_MSS;
+  int offset;
+  size_t len;
+  int i = 0;
+  FILE *f;
+#if LWIP_TCP_TIMESTAMPS
+  /* when timestamps are used, usable space is 12 bytes less per segment */
+  chunk_size -= 12;
+#endif
+
+  fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE);
+  fprintf(struct_file, "const struct fsdata_chksum chksums_%s[] = {" NEWLINE, varname);
+
+  memset(file_buffer_raw, 0xab, sizeof(file_buffer_raw));
+  f = fopen(filename, "rb");
+  if (f == INVALID_HANDLE_VALUE) {
+    printf("Failed to open file \"%s\"\n", filename);
+    exit(-1);
+  }
+  if (hdr_len > 0) {
+    /* add checksum for HTTP header */
+    fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, 0, hdr_chksum, hdr_len);
+    i++;
+  }
+  for (offset = hdr_len; ; offset += len) {
+    unsigned short chksum;
+    len = fread(file_buffer_raw, 1, chunk_size, f);
+    if (len == 0) {
+      break;
+    }
+    chksum = ~inet_chksum(file_buffer_raw, (u16_t)len);
+    /* add checksum for data */
+    fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, offset, chksum, len);
+    i++;
+  }
+  fclose(f);
+  fprintf(struct_file, "};" NEWLINE);
+  fprintf(struct_file, "#endif /* HTTPD_PRECALCULATED_CHECKSUM */" NEWLINE);
+  return i;
 }
 
 int process_file(FILE *data_file, FILE *struct_file, const char *filename)
@@ -300,6 +380,9 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
   int i = 0;
   char qualifiedName[MAX_PATH_LEN];
   int file_size;
+  u16_t http_hdr_chksum = 0;
+  u16_t http_hdr_len = 0;
+  int chksum_count;
 
   /* create qualified name (TODO: prepend slash or not?) */
   sprintf(qualifiedName,"%s/%s", curSubdir, filename);
@@ -326,19 +409,29 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
 #endif /* ALIGN_PAYLOAD */
   fprintf(data_file, NEWLINE);
 
+  file_size = get_file_size(filename);
+  if (includeHttpHeader) {
+    file_write_http_header(data_file, filename, file_size, &http_hdr_len, &http_hdr_chksum);
+  }
+  if (precalcChksum) {
+    chksum_count = write_checksums(struct_file, filename, varname, http_hdr_len, http_hdr_chksum);
+  }
+
   /* build declaration of struct fsdata_file in temp file */
   fprintf(struct_file, "const struct fsdata_file file_%s[] = { {" NEWLINE, varname);
   fprintf(struct_file, "file_%s," NEWLINE, lastFileVar);
   fprintf(struct_file, "data_%s," NEWLINE, varname);
   fprintf(struct_file, "data_%s + %d," NEWLINE, varname, i);
   fprintf(struct_file, "sizeof(data_%s) - %d," NEWLINE, varname, i);
-  fprintf(struct_file, "%d}};" NEWLINE NEWLINE, includeHttpHeader);
+  fprintf(struct_file, "%d," NEWLINE, includeHttpHeader);
+  if (precalcChksum) {
+    fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE);
+    fprintf(struct_file, "%d, chksums_%s," NEWLINE, chksum_count, varname);
+    fprintf(struct_file, "#endif /* HTTPD_PRECALCULATED_CHECKSUM */" NEWLINE);
+  }
+  fprintf(struct_file, "}};" NEWLINE NEWLINE);
   strcpy(lastFileVar, varname);
 
-  file_size = get_file_size(filename);
-  if (includeHttpHeader) {
-    file_write_http_header(data_file, filename, file_size);
-  }
   /* write actual file contents */
   i = 0;
   fprintf(data_file, NEWLINE "/* raw file data (%d bytes) */" NEWLINE, file_size);
@@ -348,91 +441,122 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename)
   return 0;
 }
 
-int file_write_http_header(FILE *data_file, const char *filename, int file_size)
+int file_write_http_header(FILE *data_file, const char *filename, int file_size,
+                           u16_t *http_hdr_len, u16_t *http_hdr_chksum)
 {
   int i = 0;
   int response_type = HTTP_HDR_OK;
   int file_type;
-  const char **httpResponseText = g_psHTTPHeaderStrings /*httpResponseText_1_0*/;
   const char *cur_string;
   size_t cur_len;
   int written = 0;
+  size_t hdr_len = 0;
+  u16_t acc;
+  const char *file_ext;
+  int j;
+
+  memset(hdr_buf, 0, sizeof(hdr_buf));
   
-#ifdef HTTP_11
   if (useHttp11) {
-    httpResponseText = httpResponseText_1_1;
+    response_type = HTTP_HDR_OK_11;
   }
-#endif
 
   fprintf(data_file, NEWLINE "/* HTTP header */");
   if (strstr(filename, "404")) {
     response_type = HTTP_HDR_NOT_FOUND;
+    if (useHttp11) {
+      response_type = HTTP_HDR_NOT_FOUND_11;
+    }
   }
-  cur_string = httpResponseText[response_type];
+  cur_string = g_psHTTPHeaderStrings[response_type];
   cur_len = strlen(cur_string);
   fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
   written += file_put_ascii(data_file, cur_string, cur_len, &i);
   i = 0;
+  if (precalcChksum) {
+    memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
+    hdr_len += cur_len;
+  }
 
   cur_string = serverID;
   cur_len = strlen(cur_string);
   fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
   written += file_put_ascii(data_file, cur_string, cur_len, &i);
   i = 0;
-
+  if (precalcChksum) {
-  if (strstr(filename, ".html") || strstr(filename, ".htm")) {
+    memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
-    file_type = HTTP_HDR_HTML;
+    hdr_len += cur_len;
-  } else if (strstr(filename, ".gif")) {
+  }
-    file_type = HTTP_HDR_GIF;
+
-  } else if (strstr(filename, ".png")) {
+  file_ext = filename;
-    file_type = HTTP_HDR_PNG;
+  while(strstr(file_ext, ".") != NULL) {
-  } else if (strstr(filename, ".jpeg") || strstr(filename, ".jpg")) {
+    file_ext = strstr(file_ext, ".");
-    file_type = HTTP_HDR_JPG;
+    file_ext++;
-  } else if (strstr(filename, ".bin") || strstr(filename, ".class")) {
+  }
-    file_type = HTTP_HDR_APP;
+  if((file_ext == NULL) || (*file_ext == 0)) {
-  } else if (strstr(filename, ".ra") || strstr(filename, ".ram")) {
+    printf("failed to get extension for file \"%s\", using default.", filename);
-    file_type = HTTP_HDR_RA;
+  } else {
-  } else if (strstr(filename, ".js")) {
+    for(j = 0; j < NUM_HTTP_HEADERS; j++) {
-    file_type = HTTP_HDR_JS;
+      if(!strcmp(file_ext, g_psHTTPHeaders[j].extension)) {
-  } else if (strstr(filename, ".css")) {
+        file_type = g_psHTTPHeaders[j].headerIndex;
-    file_type = HTTP_HDR_CSS;
+        break;
-  } else {
+      }
-    file_type = HTTP_HDR_DEFAULT_TYPE;
+    }
+    if (j >= NUM_HTTP_HEADERS) {
+      printf("failed to get file type for extension \"%s\", using default.", file_ext);
+      file_type = HTTP_HDR_DEFAULT_TYPE;
+    }
   }
-  cur_string = /*httpContentType_header*/ g_psHTTPHeaderStrings[file_type];
-  cur_len = strlen(cur_string);
-  fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
-  written += file_put_ascii(data_file, cur_string, cur_len, &i);
-  i = 0;
 
-#ifdef HTTP_11
   if (useHttp11) {
     char intbuf[MAX_PATH_LEN];
     memset(intbuf, 0, sizeof(intbuf));
 
-    cur_string = httpContentLength;
+    cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONTENT_LENGTH];
     cur_len = strlen(cur_string);
     fprintf(data_file, NEWLINE "/* \"%s%d\r\n\" (%d+ bytes) */" NEWLINE, cur_string, file_size, cur_len+2);
     written += file_put_ascii(data_file, cur_string, cur_len, &i);
-    _itoa(file_size, intbuf, 10);
+    if (precalcChksum) {
-    written += file_put_ascii(data_file, intbuf, strlen(intbuf), &i);
+      memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
-    written += file_put_ascii(data_file, "\r\n", 2, &i);
+      hdr_len += cur_len;
-    i = 0;
+    }
 
-    cur_string = httpConnectionClose;
+    _itoa(file_size, intbuf, 10);
+    strcat(intbuf, "\r\n");
+    cur_len = strlen(intbuf);
+    written += file_put_ascii(data_file, intbuf, cur_len, &i);
+    i = 0;
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], intbuf, cur_len);
+      hdr_len += cur_len;
+    }
+
+    cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONN_CLOSE];
     cur_len = strlen(cur_string);
     fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
     written += file_put_ascii(data_file, cur_string, cur_len, &i);
+    i = 0;
+    if (precalcChksum) {
+      memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
+      hdr_len += cur_len;
+    }
   }
-#else
-  LWIP_UNUSED_ARG(file_size);
-#endif
 
-  /* empty line already included in content-type line */
+  cur_string = g_psHTTPHeaderStrings[file_type];
-#if 0
+  cur_len = strlen(cur_string);
-  fprintf(data_file, NEWLINE "/* Empty line (end of header - 2 bytes) */" NEWLINE, cur_string);
+  fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len);
-  written += file_put_ascii(data_file, "\r\n", 2, &i);
+  written += file_put_ascii(data_file, cur_string, cur_len, &i);
-#endif
+  i = 0;
+  if (precalcChksum) {
+    memcpy(&hdr_buf[hdr_len], cur_string, cur_len);
+    hdr_len += cur_len;
+
+    LWIP_ASSERT("hdr_len <= 0xffff", hdr_len <= 0xffff);
+    LWIP_ASSERT("strlen(hdr_buf) == hdr_len", strlen(hdr_buf) == hdr_len);
+    acc = ~inet_chksum(hdr_buf, (u16_t)hdr_len);
+    *http_hdr_len = (u16_t)hdr_len;
+    *http_hdr_chksum = acc;
+  }
 
   return written;
 }
@@ -449,3 +573,19 @@ int file_put_ascii(FILE *file, const char* ascii_string, int len, int *i)
   }
   return len;
 }
+
+int s_put_ascii(char *buf, const char *ascii_string, int len, int *i)
+{
+  int x;
+  int idx = 0;
+  for(x = 0; x < len; x++) {
+    unsigned char cur = ascii_string[x];
+    sprintf(&buf[idx], "0x%02.2x,", cur);
+    idx += 5;
+    if ((++(*i) % HEX_BYTES_PER_LINE) == 0) {
+      sprintf(&buf[idx], NEWLINE);
+      idx += NEWLINE_LEN;
+    }
+  }
+  return len;
+}