From 591aef2ebeb9d916cf2c3d9cb809eb14cd34b6c7 Mon Sep 17 00:00:00 2001 From: Castulo Martinez Date: Wed, 17 Apr 2019 21:59:48 +0000 Subject: [PATCH] Report progress based on downloaded content When swupd reports progress of content it needs, it does so by counting how many files it needs to download (fullfiles or packs) and how many it has already downloaded. This gives a rough estimate of what is the progress of the overall download, but it can also be very misleading since some files may be very different in size compared to others. This is specially true when talking about packs, one pack could be a couple of megabytes big while another one could be a few hundred megabytes. This commit adds a curl callback that can be used to report download progress periodically based on how many bytes have been downloaded vs how many bytes have to be downloaded in total, giving the ability to report progress accurately. Signed-off-by: Castulo Martinez --- Makefile.am | 2 ++ src/curl_async.c | 49 ++++++++++++++++++++++++++++++++++++------- src/lib/progress.c | 4 ++-- src/lib/progress.h | 2 +- src/swupd.h | 1 + src/swupd_curl.h | 16 ++++++++++++++ src/swupd_progress.c | 50 ++++++++++++++++++++++++++++++++++++++++++++ src/swupd_progress.h | 27 ++++++++++++++++++++++++ 8 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 src/swupd_progress.c create mode 100644 src/swupd_progress.h diff --git a/Makefile.am b/Makefile.am index cff2c3f3..3de3cde3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -94,6 +94,8 @@ swupd_SOURCES = \ src/swupd_curl_internal.h \ src/swupd_exit_codes.h \ src/swupd_internal.h \ + src/swupd_progress.c \ + src/swupd_progress.h \ src/telemetry.c \ src/timelist.c \ src/timelist.h \ diff --git a/src/curl_async.c b/src/curl_async.c index 253b3122..8b6864a0 100644 --- a/src/curl_async.c +++ b/src/curl_async.c @@ -86,13 +86,15 @@ struct swupd_curl_parallel_handle { bool resume_failed; int last_retry; /* keep the largest retry number so far */ - CURLM *mcurl; /* Curl handle */ - struct list *failed; /* List of failed downloads */ - struct hashmap *curl_hashmap; /* Hashmap mentioned above */ - struct tp *thpool; /* Pointer to the threadpool */ - swupd_curl_success_cb success_cb; /* Callback to success function */ - swupd_curl_error_cb error_cb; /* Callback to error function */ - swupd_curl_free_cb free_cb; /* Callback to free user data*/ + CURLM *mcurl; /* Curl handle */ + struct list *failed; /* List of failed downloads */ + struct hashmap *curl_hashmap; /* Hashmap mentioned above */ + struct tp *thpool; /* Pointer to the threadpool */ + swupd_curl_success_cb success_cb; /* Callback to success function */ + swupd_curl_error_cb error_cb; /* Callback to error function */ + swupd_curl_free_cb free_cb; /* Callback to free user data */ + swupd_curl_progress_cb progress_cb; /* Callback to report download progress */ + void *data; }; /* @@ -224,6 +226,20 @@ void swupd_curl_parallel_download_set_callbacks(void *handle, swupd_curl_success h->free_cb = free_cb; } +void swupd_curl_parallel_download_set_progress_callbacks(void *handle, swupd_curl_progress_cb progress_cb, void *data) +{ + struct swupd_curl_parallel_handle *h; + + if (!handle) { + error("Curl - Invalid parallel download handle\n"); + return; + } + h = handle; + + h->progress_cb = progress_cb; + h->data = data; +} + // Try to process at most COUNT messages from the curl multi-stack. static int perform_curl_io_and_complete(struct swupd_curl_parallel_handle *h, int count) { @@ -425,6 +441,25 @@ static int process_download(struct swupd_curl_parallel_handle *h, struct multi_c } file->curl = curl; + if (h->progress_cb && h->data) { + + curl_ret = curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, h->progress_cb); + if (curl_ret != CURLE_OK) { + goto out_bad; + } + + /* switch on the progress meter */ + curl_ret = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); + if (curl_ret != CURLE_OK) { + goto out_bad; + } + + curl_ret = curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, h->data); + if (curl_ret != CURLE_OK) { + goto out_bad; + } + } + if (file->retries > 0 && !h->resume_failed && lstat(file->file.path, &stat) == 0) { info("Curl - Resuming download for '%s'\n", file->url); curl_ret = curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)stat.st_size); diff --git a/src/lib/progress.c b/src/lib/progress.c index f6a514a0..da7f987e 100644 --- a/src/lib/progress.c +++ b/src/lib/progress.c @@ -87,7 +87,7 @@ void progress_complete_step(void) } } -void progress_report(unsigned int count, unsigned int max) +void progress_report(double count, double max) { static int last_percentage = -1; static unsigned int last_step = 0; @@ -96,7 +96,7 @@ void progress_report(unsigned int count, unsigned int max) if (max != 0) { /* Only print when the percentage changes, so a maximum of 100 times per run */ - int percentage = (int)(100 * ((float)count / (float)max)); + int percentage = (int)(100 * (count / max)); if (percentage != last_percentage || step.current != last_step) { if (progress_function) { progress_function(step.description, step.current, step.total, percentage); diff --git a/src/lib/progress.h b/src/lib/progress.h index 2009c392..2184d669 100644 --- a/src/lib/progress.h +++ b/src/lib/progress.h @@ -59,7 +59,7 @@ void progress_complete_step(void); /* * It reports the partial progress of the step. Useful in long running steps. */ -void progress_report(unsigned int, unsigned int); +void progress_report(double, double); #ifdef __cplusplus } diff --git a/src/swupd.h b/src/swupd.h index 8e49bbba..5ca6e7b5 100644 --- a/src/swupd.h +++ b/src/swupd.h @@ -22,6 +22,7 @@ #include "scripts.h" #include "swupd_curl.h" #include "swupd_exit_codes.h" +#include "swupd_progress.h" #include "timelist.h" #ifdef __cplusplus diff --git a/src/swupd_curl.h b/src/swupd_curl.h index e5f5fb01..fd9b49b3 100644 --- a/src/swupd_curl.h +++ b/src/swupd_curl.h @@ -1,6 +1,7 @@ #ifndef __SWUPD_CURL__ #define __SWUPD_CURL__ +#include "swupd_progress.h" #include #ifdef __cplusplus @@ -45,6 +46,11 @@ typedef bool (*swupd_curl_error_cb)(enum download_status status, void *data); */ typedef void (*swupd_curl_free_cb)(void *data); +/* + * Callback called periodically by curl to report on download progress + */ +typedef int (*swupd_curl_progress_cb)(void *clientp, int64_t dltotal, int64_t dlnow, int64_t ultotal, int64_t ulnow); + /* * Init the swupd curl. * Must be called before any curl operation. @@ -104,6 +110,16 @@ void *swupd_curl_parallel_download_start(size_t max_xfer); */ void swupd_curl_parallel_download_set_callbacks(void *handle, swupd_curl_success_cb success_cb, swupd_curl_error_cb error_cb, swupd_curl_free_cb free_cb); +/* + * Set parallel downloads progress callback + * + * - progress_cb(): Called periodically by curl while downloading/uploading files. + * The return value of the function is not really important, the + * function prints the download progress, so it will always return 0. + * - data: User data to be informed to progress_cb. + */ +void swupd_curl_parallel_download_set_progress_callbacks(void *handle, swupd_curl_progress_cb progress_cb, void *data); + /* * Enqueue a file to be downloaded. If the number of current downloads is higher * than max_xfer, this function will be blocked for downloads until the number of diff --git a/src/swupd_progress.c b/src/swupd_progress.c new file mode 100644 index 00000000..ed904190 --- /dev/null +++ b/src/swupd_progress.c @@ -0,0 +1,50 @@ +/* + * Software Updater - client side + * + * Copyright © 2012-2019 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +#include "swupd_progress.h" +#include "swupd.h" + +int swupd_progress_callback(void *clientp, int64_t dltotal, int64_t dlnow, int64_t UNUSED_PARAM ultotal, int64_t UNUSED_PARAM ulnow) +{ + struct download_progress *download_progress; + double increment; + + download_progress = clientp; + + if (download_progress->total_download_size == 0) { + return 0; + } + + /* calculate the downloaded data size since the last + * time the function was called */ + if (dlnow < download_progress->dlprev) { + /* new file */ + download_progress->dlprev = 0; + } + increment = dlnow - download_progress->dlprev; + download_progress->dlprev = dlnow; + download_progress->current += increment; + + /* if more data has been downloaded, report progress */ + if (dltotal && increment > 0) { + progress_report(download_progress->current, download_progress->total_download_size); + } + + return 0; +} diff --git a/src/swupd_progress.h b/src/swupd_progress.h new file mode 100644 index 00000000..0719e87f --- /dev/null +++ b/src/swupd_progress.h @@ -0,0 +1,27 @@ +#ifndef __SWUPD_PROGRESS__ +#define __SWUPD_PROGRESS__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define UNUSED_PARAM __attribute__((__unused__)) + +#include + +struct download_progress { + double total_download_size; /* total number of bytes to download */ + double current; /* number of bytes that has been already downloaded */ + double dlprev; /* previous download read provided by curl */ +}; + +/* + * Callback function called periodically by CURL to report how many bytes + * it has downloaded. + */ +int swupd_progress_callback(void *clientp, int64_t dltotal, int64_t dlnow, int64_t UNUSED_PARAM ultotal, int64_t UNUSED_PARAM ulnow); + +#ifdef __cplusplus +} +#endif +#endif