[sr-dev] git:master: utils: http_query() now handles large replies properly.

[Alex Balashov]

Module: sip-router
Branch: master
Commit: 56c417b5ae29f0dd0387d59a6e9d8d203deb8855
URL:    http://git.sip-router.org/cgi-bin/gitweb.cgi/sip-router/?a=commit;h=56c417b5ae29f0dd0387d59a6e9d8d203deb8855

Author: Alex Balashov <abalashov at evaristesys.com>
Committer: Alex Balashov <abalashov at evaristesys.com>
Date:   Wed Dec 10 17:42:06 2014 -0500

utils: http_query() now handles large replies properly.

The callback function passed to CURL (CURLOPT_WRITEFUNCTION) malloc'd one buffer on the presumption that it would be called once. This is not necessarily true for large replies, so the callback was modified to realloc() the buffer upward as needed until all data chunks were retrieved.

---

 modules/utils/functions.c |   48 ++++++++++++++++++++++++++------------------
 modules/utils/utils.h     |    6 +++++
 2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/modules/utils/functions.c b/modules/utils/functions.c
index 9c23350..48fa0ff 100644
--- a/modules/utils/functions.c
+++ b/modules/utils/functions.c
@@ -46,24 +46,30 @@
 /* 
  * curl write function that saves received data as zero terminated
  * to stream. Returns the amount of data taken care of.
+ *
+ * This function may be called multiple times for larger responses, 
+ * so it reallocs + concatenates the buffer as needed.
  */
-size_t write_function( void *ptr, size_t size, size_t nmemb, void *stream)
+size_t write_function( void *ptr, size_t size, size_t nmemb, void *stream_ptr)
 {
-    /* Allocate memory and copy */
-    char* data;
+    http_res_stream_t *stream = (http_res_stream_t *) stream_ptr;
+
+    stream->buf = (char *) pkg_realloc(stream->buf, stream->curr_size + 
+				(size * nmemb) + 1);
 
-    data = (char*)pkg_malloc((size* nmemb) + 1);
-    if (data == NULL) {
+    if (stream->buf == NULL) {
 	LM_ERR("cannot allocate memory for stream\n");
 	return CURLE_WRITE_ERROR;
     }
 
-    memcpy(data, (char*)ptr, size* nmemb);
-    data[nmemb] = '\0';
-        
-    *((char**) stream) = data;
-    
-    return size* nmemb;
+    memcpy(&stream->buf[stream->pos], (char *) ptr, (size * nmemb));
+
+    stream->curr_size += ((size * nmemb) + 1);
+    stream->pos += (size * nmemb);
+
+    stream->buf[stream->pos + 1] = '\0';
+
+    return size * nmemb;
  }
 
 
@@ -77,12 +83,14 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     CURLcode res;  
     str value, post_value;
     char *url, *at, *post;
-    char* stream;
+    http_res_stream_t stream;
     long stat;
     pv_spec_t *dst;
     pv_value_t val;
     double download_size;
 
+    memset(&stream, 0, sizeof(http_res_stream_t));
+
     if (fixup_get_svalue(_m, (gparam_p)_url, &value) != 0) {
 	LM_ERR("cannot get page value\n");
 	return -1;
@@ -129,7 +137,6 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     curl_easy_setopt(curl, CURLOPT_NOSIGNAL, (long)1);
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)http_query_timeout);
 
-    stream = NULL;
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_function);
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
 
@@ -150,8 +157,8 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
 		}
 	
 		curl_easy_cleanup(curl);
-		if(stream)
-			pkg_free(stream);
+		if(stream.buf)
+			pkg_free(stream.buf);
 		return -1;
     }
 
@@ -159,14 +166,15 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     if ((stat >= 200) && (stat < 500)) {
 	curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &download_size);
 	LM_DBG("http_query download size: %u\n", (unsigned int)download_size);
+
 	/* search for line feed */
-	at = memchr(stream, (char)10, download_size);
+	at = memchr(stream.buf, (char)10, download_size);
 	if (at == NULL) {
 	    /* not found: use whole stream */
-	    at = stream + (unsigned int)download_size;
+	    at = stream.buf + (unsigned int)download_size;
 	}
-	val.rs.s = stream;
-	val.rs.len = at - stream;
+	val.rs.s = stream.buf;
+	val.rs.len = at - stream.buf;
 	LM_DBG("http_query result: %.*s\n", val.rs.len, val.rs.s);
 	val.flags = PV_VAL_STR;
 	dst = (pv_spec_t *)_dst;
@@ -174,6 +182,6 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     }
 	
     curl_easy_cleanup(curl);
-    pkg_free(stream);
+    pkg_free(stream.buf);
     return stat;
 }
diff --git a/modules/utils/utils.h b/modules/utils/utils.h
index a601b51..6653144 100644
--- a/modules/utils/utils.h
+++ b/modules/utils/utils.h
@@ -40,4 +40,10 @@ extern int http_query_timeout;
 extern db1_con_t *pres_dbh;
 extern db_func_t pres_dbf;
 
+typedef struct {
+	char		*buf;
+	size_t		curr_size;
+	size_t		pos;
+} http_res_stream_t;
+
 #endif /* UTILS_H */