From 93a0e58ea27a1e0d7fde5d7bf639db26a74f26f6 Mon Sep 17 00:00:00 2001 From: goldsimon Date: Thu, 18 Mar 2010 06:34:35 +0000 Subject: [PATCH] Added support for precalculated checksum and (pregenerated) HTTP/1.1 headers ("connection: close" and "content-length") --- apps/httpserver_raw/fs.c | 9 +- apps/httpserver_raw/fs.h | 27 +- apps/httpserver_raw/fsdata.c | 9 +- apps/httpserver_raw/fsdata.h | 9 +- apps/httpserver_raw/makefsdata/makefsdata.c | 258 +++++++++++++++----- 5 files changed, 243 insertions(+), 69 deletions(-) diff --git a/apps/httpserver_raw/fs.c b/apps/httpserver_raw/fs.c index 514ce1f..5d4fcc0 100644 --- a/apps/httpserver_raw/fs.c +++ b/apps/httpserver_raw/fs.c @@ -29,6 +29,7 @@ * Author: Adam Dunkels * */ +#include "lwip/opt.h" #include "lwip/def.h" #include "fs.h" #include "fsdata.h" @@ -44,7 +45,7 @@ /* Define the file system memory allocation structure. */ struct fs_table { struct fs_file file; - int inuse; + u8_t inuse; }; /* Allocate file system memory */ @@ -97,6 +98,10 @@ fs_open(const char *name) file->index = f->len; file->pextension = NULL; file->http_header_included = f->http_header_included; +#if HTTPD_PRECALCULATED_CHECKSUM + file->chksum_count = f->chksum_count; + file->chksum = f->chksum; +#endif /* HTTPD_PRECALCULATED_CHECKSUM */ return file; } } @@ -125,7 +130,7 @@ fs_read(struct fs_file *file, char *buffer, int count) read = count; } - memcpy(buffer, (file->data + file->index), read); + MEMCPY(buffer, (file->data + file->index), read); file->index += read; return(read); diff --git a/apps/httpserver_raw/fs.h b/apps/httpserver_raw/fs.h index e7343ae..6236dcf 100644 --- a/apps/httpserver_raw/fs.h +++ b/apps/httpserver_raw/fs.h @@ -32,17 +32,36 @@ #ifndef __FS_H__ #define __FS_H__ +#include "lwip/opt.h" + +/** HTTPD_PRECALCULATED_CHECKSUM==1: include precompiled checksums for + * predefined (MSS-sized) chunks of the files to prevent having to calculate + * the checksums at runtime. */ +#ifndef HTTPD_PRECALCULATED_CHECKSUM +#define HTTPD_PRECALCULATED_CHECKSUM 0 +#endif + +#if HTTPD_PRECALCULATED_CHECKSUM +struct fsdata_chksum { + u32_t offset; + u16_t chksum; + u16_t len; +}; +#endif /* HTTPD_PRECALCULATED_CHECKSUM */ + struct fs_file { const char *data; int len; int index; void *pextension; - int http_header_included; +#if HTTPD_PRECALCULATED_CHECKSUM + const struct fsdata_chksum *chksum; + u16_t chksum_count; +#endif /* HTTPD_PRECALCULATED_CHECKSUM */ + u8_t http_header_included; }; -/* file must be allocated by caller and will be filled in - by the function. */ -struct fs_file * fs_open(const char *name); +struct fs_file *fs_open(const char *name); void fs_close(struct fs_file *file); int fs_read(struct fs_file *file, char *buffer, int count); int fs_bytes_left(struct fs_file *file); diff --git a/apps/httpserver_raw/fsdata.c b/apps/httpserver_raw/fsdata.c index 8cc5b61..9e64666 100644 --- a/apps/httpserver_raw/fsdata.c +++ b/apps/httpserver_raw/fsdata.c @@ -263,21 +263,24 @@ file_NULL, data__img_sics_gif, data__img_sics_gif + 16, sizeof(data__img_sics_gif) - 16, -1}}; +1, +}}; const struct fsdata_file file__404_html[] = { { file__img_sics_gif, data__404_html, data__404_html + 12, sizeof(data__404_html) - 12, -1}}; +1, +}}; const struct fsdata_file file__index_html[] = { { file__404_html, data__index_html, data__index_html + 12, sizeof(data__index_html) - 12, -1}}; +1, +}}; #define FS_ROOT file__index_html #define FS_NUMFILES 3 diff --git a/apps/httpserver_raw/fsdata.h b/apps/httpserver_raw/fsdata.h index 5086bcd..6f6c557 100644 --- a/apps/httpserver_raw/fsdata.h +++ b/apps/httpserver_raw/fsdata.h @@ -32,12 +32,19 @@ #ifndef __FSDATA_H__ #define __FSDATA_H__ +#include "lwip/opt.h" +#include "fs.h" + struct fsdata_file { const struct fsdata_file *next; const unsigned char *name; const unsigned char *data; int len; - int http_header_included; + u8_t http_header_included; +#if HTTPD_PRECALCULATED_CHECKSUM + u16_t chksum_count; + const struct fsdata_chksum *chksum; +#endif /* HTTPD_PRECALCULATED_CHECKSUM */ }; #endif /* __FSDATA_H__ */ diff --git a/apps/httpserver_raw/makefsdata/makefsdata.c b/apps/httpserver_raw/makefsdata/makefsdata.c index dc8a680..c8c3817 100644 --- a/apps/httpserver_raw/makefsdata/makefsdata.c +++ b/apps/httpserver_raw/makefsdata/makefsdata.c @@ -6,6 +6,9 @@ * Author: Jim Pettinato * Simon Goldschmidt * + * @todo: + * - take TCP_MSS, LWIP_TCP_TIMESTAMPS and + * PAYLOAD_ALIGN_TYPE/PAYLOAD_ALIGNMENT as arguments */ #include @@ -60,6 +63,9 @@ #define LWIP_HTTPD_DYNAMIC_HEADERS 1 #include "../httpd_structs.h" +#include "../../../../lwip/src/core/ipv4/inet_chksum.c" +#include "../../../../lwip/src/core/def.c" + /** (Your server name here) */ const char *serverID = "Server: "HTTPD_SERVER_AGENT"\r\n"; @@ -75,12 +81,14 @@ static int payload_alingment_dummy_counter = 0; #define MAX_PATH_LEN 256 -#define COPY_BUFSIZE 1024 +#define COPY_BUFSIZE 10240 int process_sub(FILE *data_file, FILE *struct_file); int process_file(FILE *data_file, FILE *struct_file, const char *filename); -int file_write_http_header(FILE *data_file, const char *filename, int file_size); +int file_write_http_header(FILE *data_file, const char *filename, int file_size, + u16_t *http_hdr_len, u16_t *http_hdr_chksum); int file_put_ascii(FILE *file, const char *ascii_string, int len, int *i); +int s_put_ascii(char *buf, const char *ascii_string, int len, int *i); void concat_files(const char *file1, const char *file2, const char *targetfile); static unsigned char file_buffer_raw[COPY_BUFSIZE]; @@ -89,10 +97,12 @@ static char file_buffer_c[COPY_BUFSIZE * 5 + ((COPY_BUFSIZE / HEX_BYTES_PER_LINE char curSubdir[MAX_PATH_LEN]; char lastFileVar[MAX_PATH_LEN]; +char hdr_buf[4096]; unsigned char processSubs = 1; unsigned char includeHttpHeader = 1; unsigned char useHttp11 = 0; +unsigned char precalcChksum = 0; int main(int argc, char *argv[]) { @@ -120,6 +130,8 @@ int main(int argc, char *argv[]) includeHttpHeader = 0; } else if (strstr(argv[i], "-11")) { useHttp11 = 1; + } else if (strstr(argv[i], "-c")) { + precalcChksum = 1; } else { strcpy(path, argv[i]); } @@ -133,8 +145,9 @@ int main(int argc, char *argv[]) printf(" Usage: htmlgen [targetdir] [-s] [-i]" NEWLINE NEWLINE); printf(" targetdir: relative or absolute path to files to convert" NEWLINE); printf(" switch -s: toggle processing of subdirectories (default is on)" NEWLINE); - printf(" switch -e: exclude HTTP header from file (header is created at runtime, default is on)" NEWLINE); + printf(" switch -e: exclude HTTP header from file (header is created at runtime, default is off)" NEWLINE); printf(" switch -11: include HTTP 1.1 header (1.0 is default)" NEWLINE); + printf(" switch -c: precalculate checksums for all pages (default is off)" NEWLINE); printf(" if targetdir not specified, htmlgen will attempt to" NEWLINE); printf(" process files in subdirectory 'fs'" NEWLINE); exit(-1); @@ -154,7 +167,15 @@ int main(int argc, char *argv[]) GETCWD(appPath, MAX_PATH_LEN); data_file = fopen("fsdata.tmp", "wb"); + if (data_file == NULL) { + printf("Failed to create file \"fsdata.tmp\"\n"); + exit(-1); + } struct_file = fopen("fshdr.tmp", "wb"); + if (struct_file == NULL) { + printf("Failed to create file \"fshdr.tmp\"\n"); + exit(-1); + } CHDIR(path); @@ -182,6 +203,10 @@ int main(int argc, char *argv[]) printf(NEWLINE "Creating target file..." NEWLINE NEWLINE); concat_files("fsdata.tmp", "fshdr.tmp", "fsdata.c"); + /* if succeeded, delete the temporary files */ + remove("fsdata.tmp"); + remove("fshdr.tmp"); + printf(NEWLINE "Processed %d files - done." NEWLINE NEWLINE, filesProcessed); return 0; @@ -191,7 +216,11 @@ static void copy_file(const char *filename_in, FILE *fout) { FILE *fin; size_t len; - fin = fopen(filename_in, "r"); + fin = fopen(filename_in, "rb"); + if (fin == NULL) { + printf("Failed to open file \"%s\"\n", filename_in); + exit(-1); + } while((len = fread(file_buffer_raw, 1, COPY_BUFSIZE, fin)) > 0) { @@ -204,6 +233,10 @@ void concat_files(const char *file1, const char *file2, const char *targetfile) { FILE *fout; fout = fopen(targetfile, "wb"); + if (fout == NULL) { + printf("Failed to open file \"%s\"\n", targetfile); + exit(-1); + } copy_file(file1, fout); copy_file(file2, fout); fclose(fout); @@ -261,11 +294,13 @@ int get_file_size(const char* filename) FILE *inFile; int file_size = -1; inFile = fopen(filename, "rb"); - if(inFile) { - fseek (inFile , 0 , SEEK_END); - file_size = ftell(inFile); - fclose(inFile); + if (inFile == NULL) { + printf("Failed to open file \"%s\"\n", filename); + exit(-1); } + fseek(inFile, 0, SEEK_END); + file_size = ftell(inFile); + fclose(inFile); return file_size; } @@ -290,7 +325,52 @@ void process_file_data(const char *filename, FILE *data_file) } written = fwrite(file_buffer_c, 1, off, data_file); } - }while(len > 0); + } while(len > 0); + fclose(source_file); +} + +int write_checksums(FILE *struct_file, const char *filename, const char *varname, + u16_t hdr_len, u16_t hdr_chksum) +{ + int chunk_size = TCP_MSS; + int offset; + size_t len; + int i = 0; + FILE *f; +#if LWIP_TCP_TIMESTAMPS + /* when timestamps are used, usable space is 12 bytes less per segment */ + chunk_size -= 12; +#endif + + fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE); + fprintf(struct_file, "const struct fsdata_chksum chksums_%s[] = {" NEWLINE, varname); + + memset(file_buffer_raw, 0xab, sizeof(file_buffer_raw)); + f = fopen(filename, "rb"); + if (f == INVALID_HANDLE_VALUE) { + printf("Failed to open file \"%s\"\n", filename); + exit(-1); + } + if (hdr_len > 0) { + /* add checksum for HTTP header */ + fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, 0, hdr_chksum, hdr_len); + i++; + } + for (offset = hdr_len; ; offset += len) { + unsigned short chksum; + len = fread(file_buffer_raw, 1, chunk_size, f); + if (len == 0) { + break; + } + chksum = ~inet_chksum(file_buffer_raw, (u16_t)len); + /* add checksum for data */ + fprintf(struct_file, "{%d, 0x%04x, %d}," NEWLINE, offset, chksum, len); + i++; + } + fclose(f); + fprintf(struct_file, "};" NEWLINE); + fprintf(struct_file, "#endif /* HTTPD_PRECALCULATED_CHECKSUM */" NEWLINE); + return i; } int process_file(FILE *data_file, FILE *struct_file, const char *filename) @@ -300,6 +380,9 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename) int i = 0; char qualifiedName[MAX_PATH_LEN]; int file_size; + u16_t http_hdr_chksum = 0; + u16_t http_hdr_len = 0; + int chksum_count; /* create qualified name (TODO: prepend slash or not?) */ sprintf(qualifiedName,"%s/%s", curSubdir, filename); @@ -326,19 +409,29 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename) #endif /* ALIGN_PAYLOAD */ fprintf(data_file, NEWLINE); + file_size = get_file_size(filename); + if (includeHttpHeader) { + file_write_http_header(data_file, filename, file_size, &http_hdr_len, &http_hdr_chksum); + } + if (precalcChksum) { + chksum_count = write_checksums(struct_file, filename, varname, http_hdr_len, http_hdr_chksum); + } + /* build declaration of struct fsdata_file in temp file */ fprintf(struct_file, "const struct fsdata_file file_%s[] = { {" NEWLINE, varname); fprintf(struct_file, "file_%s," NEWLINE, lastFileVar); fprintf(struct_file, "data_%s," NEWLINE, varname); fprintf(struct_file, "data_%s + %d," NEWLINE, varname, i); fprintf(struct_file, "sizeof(data_%s) - %d," NEWLINE, varname, i); - fprintf(struct_file, "%d}};" NEWLINE NEWLINE, includeHttpHeader); + fprintf(struct_file, "%d," NEWLINE, includeHttpHeader); + if (precalcChksum) { + fprintf(struct_file, "#if HTTPD_PRECALCULATED_CHECKSUM" NEWLINE); + fprintf(struct_file, "%d, chksums_%s," NEWLINE, chksum_count, varname); + fprintf(struct_file, "#endif /* HTTPD_PRECALCULATED_CHECKSUM */" NEWLINE); + } + fprintf(struct_file, "}};" NEWLINE NEWLINE); strcpy(lastFileVar, varname); - file_size = get_file_size(filename); - if (includeHttpHeader) { - file_write_http_header(data_file, filename, file_size); - } /* write actual file contents */ i = 0; fprintf(data_file, NEWLINE "/* raw file data (%d bytes) */" NEWLINE, file_size); @@ -348,91 +441,122 @@ int process_file(FILE *data_file, FILE *struct_file, const char *filename) return 0; } -int file_write_http_header(FILE *data_file, const char *filename, int file_size) +int file_write_http_header(FILE *data_file, const char *filename, int file_size, + u16_t *http_hdr_len, u16_t *http_hdr_chksum) { int i = 0; int response_type = HTTP_HDR_OK; int file_type; - const char **httpResponseText = g_psHTTPHeaderStrings /*httpResponseText_1_0*/; const char *cur_string; size_t cur_len; int written = 0; + size_t hdr_len = 0; + u16_t acc; + const char *file_ext; + int j; + + memset(hdr_buf, 0, sizeof(hdr_buf)); -#ifdef HTTP_11 if (useHttp11) { - httpResponseText = httpResponseText_1_1; + response_type = HTTP_HDR_OK_11; } -#endif fprintf(data_file, NEWLINE "/* HTTP header */"); if (strstr(filename, "404")) { response_type = HTTP_HDR_NOT_FOUND; + if (useHttp11) { + response_type = HTTP_HDR_NOT_FOUND_11; + } } - cur_string = httpResponseText[response_type]; + cur_string = g_psHTTPHeaderStrings[response_type]; cur_len = strlen(cur_string); fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len); written += file_put_ascii(data_file, cur_string, cur_len, &i); i = 0; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], cur_string, cur_len); + hdr_len += cur_len; + } cur_string = serverID; cur_len = strlen(cur_string); fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len); written += file_put_ascii(data_file, cur_string, cur_len, &i); i = 0; - - if (strstr(filename, ".html") || strstr(filename, ".htm")) { - file_type = HTTP_HDR_HTML; - } else if (strstr(filename, ".gif")) { - file_type = HTTP_HDR_GIF; - } else if (strstr(filename, ".png")) { - file_type = HTTP_HDR_PNG; - } else if (strstr(filename, ".jpeg") || strstr(filename, ".jpg")) { - file_type = HTTP_HDR_JPG; - } else if (strstr(filename, ".bin") || strstr(filename, ".class")) { - file_type = HTTP_HDR_APP; - } else if (strstr(filename, ".ra") || strstr(filename, ".ram")) { - file_type = HTTP_HDR_RA; - } else if (strstr(filename, ".js")) { - file_type = HTTP_HDR_JS; - } else if (strstr(filename, ".css")) { - file_type = HTTP_HDR_CSS; - } else { - file_type = HTTP_HDR_DEFAULT_TYPE; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], cur_string, cur_len); + hdr_len += cur_len; + } + + file_ext = filename; + while(strstr(file_ext, ".") != NULL) { + file_ext = strstr(file_ext, "."); + file_ext++; + } + if((file_ext == NULL) || (*file_ext == 0)) { + printf("failed to get extension for file \"%s\", using default.", filename); + } else { + for(j = 0; j < NUM_HTTP_HEADERS; j++) { + if(!strcmp(file_ext, g_psHTTPHeaders[j].extension)) { + file_type = g_psHTTPHeaders[j].headerIndex; + break; + } + } + if (j >= NUM_HTTP_HEADERS) { + printf("failed to get file type for extension \"%s\", using default.", file_ext); + file_type = HTTP_HDR_DEFAULT_TYPE; + } } - cur_string = /*httpContentType_header*/ g_psHTTPHeaderStrings[file_type]; - cur_len = strlen(cur_string); - fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len); - written += file_put_ascii(data_file, cur_string, cur_len, &i); - i = 0; -#ifdef HTTP_11 if (useHttp11) { char intbuf[MAX_PATH_LEN]; memset(intbuf, 0, sizeof(intbuf)); - cur_string = httpContentLength; + cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONTENT_LENGTH]; cur_len = strlen(cur_string); fprintf(data_file, NEWLINE "/* \"%s%d\r\n\" (%d+ bytes) */" NEWLINE, cur_string, file_size, cur_len+2); written += file_put_ascii(data_file, cur_string, cur_len, &i); - _itoa(file_size, intbuf, 10); - written += file_put_ascii(data_file, intbuf, strlen(intbuf), &i); - written += file_put_ascii(data_file, "\r\n", 2, &i); - i = 0; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], cur_string, cur_len); + hdr_len += cur_len; + } - cur_string = httpConnectionClose; + _itoa(file_size, intbuf, 10); + strcat(intbuf, "\r\n"); + cur_len = strlen(intbuf); + written += file_put_ascii(data_file, intbuf, cur_len, &i); + i = 0; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], intbuf, cur_len); + hdr_len += cur_len; + } + + cur_string = g_psHTTPHeaderStrings[HTTP_HDR_CONN_CLOSE]; cur_len = strlen(cur_string); fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len); written += file_put_ascii(data_file, cur_string, cur_len, &i); + i = 0; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], cur_string, cur_len); + hdr_len += cur_len; + } } -#else - LWIP_UNUSED_ARG(file_size); -#endif - /* empty line already included in content-type line */ -#if 0 - fprintf(data_file, NEWLINE "/* Empty line (end of header - 2 bytes) */" NEWLINE, cur_string); - written += file_put_ascii(data_file, "\r\n", 2, &i); -#endif + cur_string = g_psHTTPHeaderStrings[file_type]; + cur_len = strlen(cur_string); + fprintf(data_file, NEWLINE "/* \"%s\" (%d bytes) */" NEWLINE, cur_string, cur_len); + written += file_put_ascii(data_file, cur_string, cur_len, &i); + i = 0; + if (precalcChksum) { + memcpy(&hdr_buf[hdr_len], cur_string, cur_len); + hdr_len += cur_len; + + LWIP_ASSERT("hdr_len <= 0xffff", hdr_len <= 0xffff); + LWIP_ASSERT("strlen(hdr_buf) == hdr_len", strlen(hdr_buf) == hdr_len); + acc = ~inet_chksum(hdr_buf, (u16_t)hdr_len); + *http_hdr_len = (u16_t)hdr_len; + *http_hdr_chksum = acc; + } return written; } @@ -449,3 +573,19 @@ int file_put_ascii(FILE *file, const char* ascii_string, int len, int *i) } return len; } + +int s_put_ascii(char *buf, const char *ascii_string, int len, int *i) +{ + int x; + int idx = 0; + for(x = 0; x < len; x++) { + unsigned char cur = ascii_string[x]; + sprintf(&buf[idx], "0x%02.2x,", cur); + idx += 5; + if ((++(*i) % HEX_BYTES_PER_LINE) == 0) { + sprintf(&buf[idx], NEWLINE); + idx += NEWLINE_LEN; + } + } + return len; +}