Report progress based on downloaded content

When swupd reports progress of content it needs, it does so by counting
how many files it needs to download (fullfiles or packs) and how many it
has already downloaded. This gives a rough estimate of what is the
progress of the overall download, but it can also be very misleading
since some files may be very different in size compared to others. This
is specially true when talking about packs, one pack could be a couple
of megabytes big while another one could be a few hundred megabytes.

This commit adds a curl callback that can be used to report download
progress periodically based on how many bytes have been downloaded vs
how many bytes have to be downloaded in total, giving the ability to report
progress accurately.

Signed-off-by: Castulo Martinez <castulo.martinez@intel.com>

Makefile.am

@@ -94,6 +94,8 @@ swupd_SOURCES = \
 	src/swupd_curl_internal.h \
 	src/swupd_exit_codes.h \
 	src/swupd_internal.h \
+	src/swupd_progress.c \
+	src/swupd_progress.h \
 	src/telemetry.c \
 	src/timelist.c \
 	src/timelist.h \

src/curl_async.c

@@ -86,13 +86,15 @@ struct swupd_curl_parallel_handle {
 	bool resume_failed;
 	int last_retry; /* keep the largest retry number so far */
 
-	CURLM *mcurl;			  /* Curl handle */
-	struct list *failed;		  /* List of failed downloads */
-	struct hashmap *curl_hashmap;     /* Hashmap mentioned above */
-	struct tp *thpool;		  /* Pointer to the threadpool */
-	swupd_curl_success_cb success_cb; /* Callback to success function */
-	swupd_curl_error_cb error_cb;     /* Callback to error function */
-	swupd_curl_free_cb free_cb;       /* Callback to free user data*/
+	CURLM *mcurl;			    /* Curl handle */
+	struct list *failed;		    /* List of failed downloads */
+	struct hashmap *curl_hashmap;       /* Hashmap mentioned above */
+	struct tp *thpool;		    /* Pointer to the threadpool */
+	swupd_curl_success_cb success_cb;   /* Callback to success function */
+	swupd_curl_error_cb error_cb;       /* Callback to error function */
+	swupd_curl_free_cb free_cb;	 /* Callback to free user data */
+	swupd_curl_progress_cb progress_cb; /* Callback to report download progress */
+	void *data;
 };
 
 /*
@@ -224,6 +226,20 @@ void swupd_curl_parallel_download_set_callbacks(void *handle, swupd_curl_success
 	h->free_cb = free_cb;
 }
 
+void swupd_curl_parallel_download_set_progress_callbacks(void *handle, swupd_curl_progress_cb progress_cb, void *data)
+{
+	struct swupd_curl_parallel_handle *h;
+
+	if (!handle) {
+		error("Curl - Invalid parallel download handle\n");
+		return;
+	}
+	h = handle;
+
+	h->progress_cb = progress_cb;
+	h->data = data;
+}
+
 // Try to process at most COUNT messages from the curl multi-stack.
 static int perform_curl_io_and_complete(struct swupd_curl_parallel_handle *h, int count)
 {
@@ -425,6 +441,25 @@ static int process_download(struct swupd_curl_parallel_handle *h, struct multi_c
 	}
 	file->curl = curl;
 
+	if (h->progress_cb && h->data) {
+
+		curl_ret = curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, h->progress_cb);
+		if (curl_ret != CURLE_OK) {
+			goto out_bad;
+		}
+
+		/* switch on the progress meter */
+		curl_ret = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+		if (curl_ret != CURLE_OK) {
+			goto out_bad;
+		}
+
+		curl_ret = curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, h->data);
+		if (curl_ret != CURLE_OK) {
+			goto out_bad;
+		}
+	}
+
 	if (file->retries > 0 && !h->resume_failed && lstat(file->file.path, &stat) == 0) {
 		info("Curl - Resuming download for '%s'\n", file->url);
 		curl_ret = curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)stat.st_size);

src/lib/progress.c

@@ -87,7 +87,7 @@ void progress_complete_step(void)
 	}
 }
 
-void progress_report(unsigned int count, unsigned int max)
+void progress_report(double count, double max)
 {
 	static int last_percentage = -1;
 	static unsigned int last_step = 0;
@@ -96,7 +96,7 @@ void progress_report(unsigned int count, unsigned int max)
 	if (max != 0) {
 
 		/* Only print when the percentage changes, so a maximum of 100 times per run */
-		int percentage = (int)(100 * ((float)count / (float)max));
+		int percentage = (int)(100 * (count / max));
 		if (percentage != last_percentage || step.current != last_step) {
 			if (progress_function) {
 				progress_function(step.description, step.current, step.total, percentage);

src/lib/progress.h

@@ -59,7 +59,7 @@ void progress_complete_step(void);
 /*
  * It reports the partial progress of the step. Useful in long running steps.
  */
-void progress_report(unsigned int, unsigned int);
+void progress_report(double, double);
 
 #ifdef __cplusplus
 }

src/swupd.h

@@ -22,6 +22,7 @@
 #include "scripts.h"
 #include "swupd_curl.h"
 #include "swupd_exit_codes.h"
+#include "swupd_progress.h"
 #include "timelist.h"
 
 #ifdef __cplusplus

src/swupd_curl.h

@@ -1,6 +1,7 @@
 #ifndef __SWUPD_CURL__
 #define __SWUPD_CURL__
 
+#include "swupd_progress.h"
 #include <stdbool.h>
 
 #ifdef __cplusplus
@@ -45,6 +46,11 @@ typedef bool (*swupd_curl_error_cb)(enum download_status status, void *data);
  */
 typedef void (*swupd_curl_free_cb)(void *data);
 
+/*
+ * Callback called periodically by curl to report on download progress
+ */
+typedef int (*swupd_curl_progress_cb)(void *clientp, int64_t dltotal, int64_t dlnow, int64_t ultotal, int64_t ulnow);
+
 /*
  * Init the swupd curl.
  * Must be called before any curl operation.
@@ -104,6 +110,16 @@ void *swupd_curl_parallel_download_start(size_t max_xfer);
  */
 void swupd_curl_parallel_download_set_callbacks(void *handle, swupd_curl_success_cb success_cb, swupd_curl_error_cb error_cb, swupd_curl_free_cb free_cb);
 
+/*
+ * Set parallel downloads progress callback
+ *
+ *  - progress_cb(): Called periodically by curl while downloading/uploading files.
+ *                   The return value of the function is not really important, the
+ *                   function prints the download progress, so it will always return 0.
+ *  - data:          User data to be informed to progress_cb.
+ */
+void swupd_curl_parallel_download_set_progress_callbacks(void *handle, swupd_curl_progress_cb progress_cb, void *data);
+
 /*
  * Enqueue a file to be downloaded. If the number of current downloads is higher
  * than max_xfer, this function will be blocked for downloads until the number of

src/swupd_progress.c

@@ -0,0 +1,50 @@
+/*
+ *   Software Updater - client side
+ *
+ *      Copyright © 2012-2019 Intel Corporation.
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, version 2 or later of the License.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "swupd_progress.h"
+#include "swupd.h"
+
+int swupd_progress_callback(void *clientp, int64_t dltotal, int64_t dlnow, int64_t UNUSED_PARAM ultotal, int64_t UNUSED_PARAM ulnow)
+{
+	struct download_progress *download_progress;
+	double increment;
+
+	download_progress = clientp;
+
+	if (download_progress->total_download_size == 0) {
+		return 0;
+	}
+
+	/* calculate the downloaded data size since the last
+	 * time the function was called */
+	if (dlnow < download_progress->dlprev) {
+		/* new file */
+		download_progress->dlprev = 0;
+	}
+	increment = dlnow - download_progress->dlprev;
+	download_progress->dlprev = dlnow;
+	download_progress->current += increment;
+
+	/* if more data has been downloaded, report progress */
+	if (dltotal && increment > 0) {
+		progress_report(download_progress->current, download_progress->total_download_size);
+	}
+
+	return 0;
+}

src/swupd_progress.h

@@ -0,0 +1,27 @@
+#ifndef __SWUPD_PROGRESS__
+#define __SWUPD_PROGRESS__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UNUSED_PARAM __attribute__((__unused__))
+
+#include <curl/curl.h>
+
+struct download_progress {
+	double total_download_size; /* total number of bytes to download */
+	double current;		    /* number of bytes that has been already downloaded */
+	double dlprev;		    /* previous download read provided by curl */
+};
+
+/*
+ * Callback function called periodically by CURL to report how many bytes
+ * it has downloaded.
+ */
+int swupd_progress_callback(void *clientp, int64_t dltotal, int64_t dlnow, int64_t UNUSED_PARAM ultotal, int64_t UNUSED_PARAM ulnow);
+
+#ifdef __cplusplus
+}
+#endif
+#endif