From 15e6b1466fc8652aaa53dc86a44e002087c4ea19 Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Wed, 16 Aug 2017 15:12:11 -0700 Subject: [PATCH 1/2] Calculate entire contentsize for manifest Instead of only calculating the contentsize for files that were updated in the current version (an inaccurate number for download size since this is not the compressed size), calculate for all files in the manifest. Additionally, do not add submanifest contentsizes to the current manifest contentsize, as this will result in overcount on client systems when multiple bundles include the same bundle. With this change the contentsize field of the manifests will only report the size of the files unique to that bundle. It is then the client's responsibility to calculate total bundle size including included bundles. This is reasonably easy to accomplish with the upcoming swupd-client bundle-list --deps feature. Although the above use case is a bit more work for the client, it additionally allows a user to calculate the installation size of multiple bundles much more easily, since it only has to count bundle dependencies once to ensure files are not over counted. If two bundles not in the same include chain have overlapping content, summing the include chain of each bundle in the client will result in an over-estimation of the total size on the system. The more content is shared, the higher the over-estimation. In reality this overlap will not be large, but it is currently impossible to calculate the exact installed size using just the contentsize. Signed-off-by: Matthew Johnson --- src/manifest.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/manifest.c b/src/manifest.c index 0cb7bb7..defa2d3 100644 --- a/src/manifest.c +++ b/src/manifest.c @@ -675,19 +675,29 @@ char *file_type_to_string(struct file *file) return type; } +/* Calculate the contentsize for the manifest based on file sizes. + * + * This should calculate the files uniquely included in this manifest, but none + * of its submanifests, which will allow calculation of sizes of all bundles on + * a system by adding all manifest->contentsizes of installed bundles. + * + * However, if two bundles not in the same include chain have overlapping + * content, summing the include chain of each bundle in the client will result + * in an over-estimation of the total size on the system. The more content is + * shared, the higher the over-estimation. In reality this overlap will not be + * large, but it is currently impossible to calculate the exact installed size + * using just the contentsize. + */ static void compute_content_size(struct manifest *manifest) { - /* FIXME: this is a temporary implementation based on worst case */ - GList *list; struct file *file; - struct manifest *submanifest; list = g_list_first(manifest->files); while (list) { file = list->data; list = g_list_next(list); - if (!file->is_deleted && (file->last_change == manifest->version)) { + if (!file->is_deleted) { if (file->is_file) { manifest->contentsize += file->stat.st_size; } else if (file->is_link) { @@ -697,17 +707,6 @@ static void compute_content_size(struct manifest *manifest) } } } - - list = g_list_first(manifest->submanifests); - while (list) { - submanifest = list->data; - list = g_list_next(list); - - /* Do not take into account groups not included in download content */ - if (create_download_content_for_group(submanifest->component)) { - manifest->contentsize += submanifest->contentsize; - } - } } /* Returns 0 == success, -1 == failure */ From 350bb54fdf3852c0496fbe7d8e47c32221012236 Mon Sep 17 00:00:00 2001 From: Matthew Johnson Date: Thu, 17 Aug 2017 12:49:06 -0700 Subject: [PATCH 2/2] Add functional test for contentsize fix Add functional test to check manifest contentsize across versions and includes. Signed-off-by: Matthew Johnson --- Makefile.am | 1 + .../test.bats | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100755 test/functional/contentsize-across-versions-includes/test.bats diff --git a/Makefile.am b/Makefile.am index 1707c10..19cdfd6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,6 +108,7 @@ TESTS = $(dist_check_SCRIPTS) dist_check_SCRIPTS = \ test/functional/basic/test.bats \ + test/functional/contentsize-across-versions-includes/test.bats \ test/functional/delete-no-version-bump/test.bats \ test/functional/file-name-blacklisted/test.bats \ test/functional/format-no-decrement/test.bats \ diff --git a/test/functional/contentsize-across-versions-includes/test.bats b/test/functional/contentsize-across-versions-includes/test.bats new file mode 100755 index 0000000..d8e3f74 --- /dev/null +++ b/test/functional/contentsize-across-versions-includes/test.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats + +# common functions +load "../swupdlib" + +setup() { + clean_test_dir + init_test_dir + + init_server_ini + init_groups_ini os-core test-bundle1 test-bundle2 + + set_os_release 10 os-core + track_bundle 10 os-core + track_bundle 10 test-bundle1 + track_bundle 10 test-bundle2 + gen_file_plain 10 test-bundle1 foo + gen_file_plain 10 test-bundle1 foobar + gen_file_plain 10 test-bundle2 foo2 + gen_includes_file test-bundle2 10 test-bundle1 + + set_os_release 20 os-core + track_bundle 20 os-core + track_bundle 20 test-bundle1 + track_bundle 20 test-bundle2 + gen_file_plain 20 test-bundle1 foo + gen_file_plain 20 test-bundle1 foobar + gen_file_plain 20 test-bundle1 foobarbaz + gen_file_plain 20 test-bundle2 foo2 + gen_file_plain 20 test-bundle2 foo2bar + gen_includes_file test-bundle2 20 test-bundle1 +} + +@test "correct contentsize" { + # create a couple updates to both check that contentsize does not add included + # bundles and to verify that files changed in previous updates are counted. + sudo $CREATE_UPDATE --osversion 10 --statedir $DIR --format 3 + set_latest_ver 10 + + # contentsize for test-bundle2 should not include test-bundle1's contentsize + [[ 1 -eq $(grep '^contentsize: 11$' $DIR/www/10/Manifest.test-bundle1 | wc -l) ]] + [[ 1 -eq $(grep '^contentsize: 5$' $DIR/www/10/Manifest.test-bundle2 | wc -l) ]] + # os-core is large because it includes /usr/* + [[ 1 -eq $(grep '^contentsize: 5134$' $DIR/www/10/Manifest.os-core | wc -l) ]] + # 5134 + 11 + 5 = 5150 + [[ 1 -eq $(grep '^contentsize: 5150$' $DIR/www/10/Manifest.full | wc -l) ]] + + sudo $CREATE_UPDATE --osversion 20 --statedir $DIR --format 3 + set_latest_ver 20 + + # one new file: foobarbaz (10 bytes) + [[ 1 -eq $(grep '^contentsize: 21$' $DIR/www/20/Manifest.test-bundle1 | wc -l) ]] + # one new file: foo2bar (8 bytes) + [[ 1 -eq $(grep '^contentsize: 13$' $DIR/www/20/Manifest.test-bundle2 | wc -l) ]] + # os-core should not change size + [[ 1 -eq $(grep '^contentsize: 5134$' $DIR/www/10/Manifest.os-core | wc -l) ]] + # contentsize for full should be all files, including ones not changed in this release + # two new files: foo2bar (8 bytes) and foobarbaz (10 bytes) + # 5150 + 10 + 8 = 5168 + # 5134 + 21 + 13 = 5168 + [[ 1 -eq $(grep '^contentsize: 5168$' $DIR/www/20/Manifest.full | wc -l) ]] +} + +# vi: ft=sh ts=8 sw=2 sts=2 et tw=80