From 7eff79397c1ea872e8872ac4c8de5cc667965b74 Mon Sep 17 00:00:00 2001 From: Icarus Sparry Date: Thu, 27 Apr 2017 11:34:40 -0700 Subject: [PATCH] Rename support Signed-off-by: Icarus Sparry --- configure.ac | 5 + include/swupd.h | 15 +- src/analyze_fs.c | 6 +- src/create_update.c | 18 +-- src/fullfiles.c | 10 +- src/groups.c | 2 + src/manifest.c | 4 + src/pack.c | 2 +- src/rename.c | 371 ++++++++++++++++++++++++++------------------ 9 files changed, 259 insertions(+), 174 deletions(-) diff --git a/configure.ac b/configure.ac index 8793abe..c89bd4b 100644 --- a/configure.ac +++ b/configure.ac @@ -82,6 +82,11 @@ AS_IF([test "$enable_lzma" != "no"], [ ]) AM_CONDITIONAL([ENABLE_LZMA], [test "$enable_lzma" != "no"]) +AC_ARG_ENABLE([rename-detection], [AS_HELP_STRING([--enable-rename-detection], [enable rename detection feature])]) + +AS_IF([test "$enable_rename_detection" = "yes"], [AC_DEFINE(RENAMES,1,[Use rename detection])]) +AM_CONDITIONAL([RENAMES], [test "$enable_rename_detection" = "yes"]) + AC_CONFIG_FILES([Makefile]) AC_REQUIRE_AUX_FILE([tap-driver.sh]) AC_OUTPUT diff --git a/include/swupd.h b/include/swupd.h index a1fd405..7e8e9dc 100644 --- a/include/swupd.h +++ b/include/swupd.h @@ -65,6 +65,13 @@ #include #endif +/* Approximatly the smallest size of a pair of input files which + * differ by a single bit that bsdiff can produce a more compact + * deltafile. Files smaller than this are always marked as different. + * See the magic 200 value in the bsdiff/src/diff.c code. + */ +#define BSDIFFSIZE 200 + struct manifest { unsigned long long int format; int version; @@ -122,7 +129,7 @@ struct file { double rename_score; struct file *rename_peer; char *alpha_only_filename; /* filename minus all numerics/etc */ - char *filetype; + char *filetype; /* The output of 'file', truncated */ char *basename; char *dirname; /* end of rename detection fields */ @@ -218,8 +225,6 @@ extern void chroot_create_full(int newversion); extern void read_group_file(char *filename); extern void release_group_file(void); -extern char *group_groups(char *group); -extern char *group_packages(char *group); extern char *group_status(char *group); extern char *next_group(void); @@ -242,8 +247,8 @@ extern int previous_version_manifest(struct manifest *mom, char *name); extern void type_change_detection(struct manifest *manifest); -extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); -extern void link_renames(GList *newfiles, struct manifest *from_manifest); +extern void rename_detection(struct manifest *manifest); +extern void link_renames(GList *newfiles, int to_version); extern void __create_delta(struct file *file, int from_version, char *from_hash); extern void account_delta_hit(void); diff --git a/src/analyze_fs.c b/src/analyze_fs.c index cb3698a..f858971 100644 --- a/src/analyze_fs.c +++ b/src/analyze_fs.c @@ -42,6 +42,10 @@ static GThreadPool *threadpool; +/* Why not strcpy? Looks like the hash was going to be stored in + * binary at one stage. Should use g_string_chunk_insert_const to + * change hash_compare to a pointer compare + */ void hash_assign(char *src, char *dst) { memcpy(dst, src, SWUPD_HASH_LEN - 1); @@ -196,7 +200,7 @@ int compute_hash(struct file *file, char *filename) return 0; } - hash_set_zeros(key); + hash_set_zeros(key); /* Set to 64 '0' (not '\0') characters */ if (file->is_link) { char link[PATH_MAX]; diff --git a/src/create_update.c b/src/create_update.c index c35c844..b86d037 100644 --- a/src/create_update.c +++ b/src/create_update.c @@ -332,8 +332,6 @@ int main(int argc, char **argv) } apply_heuristics(new_full); -#warning disabled rename detection for some simplicity - // rename_detection(new_full); print_elapsed_time("full manifest creation", &previous_time, ¤t_time); @@ -365,13 +363,12 @@ int main(int argc, char **argv) apply_heuristics(new_core); /* Step 3c: ... else save the manifest */ type_change_detection(new_core); - -#warning disabled rename detection for some simplicity - /* Detect renamed files specifically for each pack */ - // rename_detection(...); - +#ifdef RENAMES + /* Detect renamed files specifically for os-core */ + rename_detection(new_core); +#endif old_deleted = remove_old_deleted_files(old_core, new_core); - sort_manifest_by_version(new_core); + sort_manifest_by_version(new_core); /* sorts by filename */ newfiles = prune_manifest(new_core); if (newfiles <= 0) { LOG(NULL, "", "Core component has not changed (after pruning), exiting"); @@ -486,7 +483,10 @@ int main(int argc, char **argv) newm->version = oldm->version; } else { apply_heuristics(newm); -#warning missing rename_detection here +#ifdef RENAMES + /* Detect renamed files specifically for this bundle */ + rename_detection(newm); +#endif /* Step 6b: otherwise, write out the manifest */ old_deleted = remove_old_deleted_files(oldm, newm); sort_manifest_by_version(newm); diff --git a/src/fullfiles.c b/src/fullfiles.c index 882e83b..985feab 100644 --- a/src/fullfiles.c +++ b/src/fullfiles.c @@ -249,22 +249,20 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest) struct file *tmp; // presort by hash for easy deduplication - manifest->files = g_list_sort(manifest->files, file_sort_hash); + list = manifest->files = g_list_sort(manifest->files, file_sort_hash); - list = g_list_first(manifest->files); - while (prev == NULL && list != NULL) { + for (; list ; list = g_list_next(list)) { tmp = list->data; - list = g_list_next(list); // find first new file if (tmp->last_change == manifest->version) { prev = tmp; outfiles = g_list_prepend(outfiles, tmp); + break; } } - while (list) { + for (; list ; list = g_list_next(list)) { file = list->data; - list = g_list_next(list); // add any new file having a unique hash //FIXME: rename logic will be needed here diff --git a/src/groups.c b/src/groups.c index 9dc4609..c01a97f 100644 --- a/src/groups.c +++ b/src/groups.c @@ -37,6 +37,7 @@ static char **groups = NULL; static gsize groupcount = 0; static unsigned int groupcursor = 0; +#if 0 char *group_packages(char *group) { assert(groupfile != NULL); @@ -50,6 +51,7 @@ char *group_groups(char *group) return g_key_file_get_value(groupfile, group, "groups", NULL); } +#endif char *group_status(char *group) { diff --git a/src/manifest.c b/src/manifest.c index 8b0c161..eab6349 100644 --- a/src/manifest.c +++ b/src/manifest.c @@ -51,6 +51,10 @@ int file_sort_hash(gconstpointer a, gconstpointer b) return memcmp(A->hash, B->hash, SWUPD_HASH_LEN - 1); } +/* Standard sort compare function which sorts + * first on the version where the file changed and then + * on the name + */ int file_sort_version(gconstpointer a, gconstpointer b) { struct file *A, *B; diff --git a/src/pack.c b/src/pack.c index 45e31eb..1d61b71 100644 --- a/src/pack.c +++ b/src/pack.c @@ -144,7 +144,7 @@ static void prepare_pack(struct packdata *pack) match_manifests(manifest, pack->end_manifest); - link_renames(pack->end_manifest->files, manifest); + link_renames(pack->end_manifest->files, pack->to); } static void make_pack_full_files(struct packdata *pack) diff --git a/src/rename.c b/src/rename.c index b4335f0..157361e 100644 --- a/src/rename.c +++ b/src/rename.c @@ -21,6 +21,10 @@ * */ + +/* Rename detection and support. + */ + #define _GNU_SOURCE #include #include @@ -37,8 +41,55 @@ #include -static magic_t mcookie; +static bool samefiletype(char *t1, char *t2) +{ + if (t1 && (t1 == t2) && *t1) { + return true; + } + return false; +} +/* For an elf binary, we get the BuildID in it, which + * is essentially a hash of the original loaded sections + * and hence is unique. This hardly makes for a 'type' + */ +static char *getmagic(char *filename) +{ + static magic_t mcookie; + static GStringChunk *typestore; + char *c2; + char *c1; + if (mcookie == NULL) { + mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS); + magic_load(mcookie, NULL); + typestore = g_string_chunk_new(200); + } + c1 = (char *)magic_file(mcookie, filename); + if (!c1) { + LOG(NULL, "Cannot find file type", "%s", filename); + c1=""; + } + c1 = strdup(c1); + c2 = strstr(c1, ", BuildID["); + if (c2) { + *c2 = 0; + } + c2 = strstr(c1, "not stripped"); + if (c2) { + *c2 = 0; + } + c2 = strstr(c1, "stripped"); + if (c2) { + *c2 = 0; + } + c2 = g_string_chunk_insert_const(typestore, c1); + free(c1); + return c2; +} + +/* Assign a score roughly in the range -100 to 1000 to express how similar + * two files are. + */ double rename_score(struct file *old, struct file *new) { double score = 0.0; @@ -50,6 +101,16 @@ double rename_score(struct file *old, struct file *new) score += 400; } + /* If the files are smaller than about 200 bytes then even a + * single byte change using bsdiff is going to work out as + * bigger than just shipping the new file, so stop if they are + * not the same. No point in running up bsdiff just for the + * sake of it. + */ + if (new->stat.st_size < BSDIFFSIZE) { + return -99.0; + } + /* points for being in the same directory */ if (strcmp(old->dirname, new->dirname) == 0) { score += 10; @@ -86,12 +147,14 @@ double rename_score(struct file *old, struct file *new) in--; } +#if 0 /* if both start with /boot/vmlinuz give it a boost; this is a local hack due to vmlinuz being very short */ if (strncmp(old->filename, "/boot/vmlinuz", 13) == 0 && strncmp(new->filename, "/boot/vmlinuz", 13) == 0) { score += 80; } /* if ELF, points for sharing the same soname to the first dot */ +#endif /* negative points for not being within 25%+/-1Kb of the same file size */ if (old->stat.st_size > ((new->stat.st_size * 1.25) + 1024)) { @@ -115,66 +178,40 @@ double rename_score(struct file *old, struct file *new) } /* negative points for not having the same 'file' type */ - if (old->filetype && new->filetype && strcmp(old->filetype, new->filetype) != 0) { + if (!samefiletype(old->filetype, new->filetype)) { score -= 60; } return score; } -static void precompute_file_data(struct manifest *manifest, struct file *file, int old_rename, GList *last_versions_list) +static void precompute_file_data(int version, const char *component, struct file *file, bool fast) { - GList *item; char *c1, *c2; char *filename = NULL; - int last_change; - struct stat buf; + assert(file); /* fill in the filename-minus-the-numbers field */ file->alpha_only_filename = calloc(strlen(file->filename) + 1, sizeof(char)); c1 = file->filename; c2 = file->alpha_only_filename; - while (*c1) { - while (*c1 && !isalpha(*c1)) - c1++; - if (!*c1) { - break; - } - if (c2 == NULL) { - break; - } - *c2 = *c1; - c1++; - c2++; - } - - if (manifest) { - string_or_die(&filename, "%s/%i/%s/%s", image_dir, manifest->version, manifest->component, file->filename); - } else if (old_rename) { - item = g_list_first(last_versions_list); - while (item) { - last_change = GPOINTER_TO_INT(item->data); - item = g_list_next(item); - - free(filename); - string_or_die(&filename, "%s/%i/full/%s", image_dir, last_change, file->filename); - if (!lstat(filename, &buf)) { - break; + if (c2) { + for(; *c1 ; c1++) { + if (isalpha(*c1)) { /* Only copy letters */ + *c2++ = *c1; } } - } else { - string_or_die(&filename, "%s/%i/full/%s", image_dir, file->last_change, file->filename); + /* alpha_only_filename is NUL terminated by calloc */ } + string_or_die(&filename, "%s/%i/%s/%s", image_dir, version, component, file->filename); /* make sure file->stat.st_size is valid */ if (file->stat.st_size == 0) { int ret; + struct stat buf; - if (filename == NULL) { - printf("filename is null...impossible to stat\n"); - assert(0); - } + assert(filename); ret = lstat(filename, &buf); if (!ret) { file->stat.st_size = buf.st_size; @@ -182,27 +219,19 @@ static void precompute_file_data(struct manifest *manifest, struct file *file, i printf("Stat failure on %s\n", filename); } } - - c1 = (char *)magic_file(mcookie, filename); - if (c1) { - char *c2; - file->filetype = strdup(c1); - c2 = strstr(file->filetype, "not stripped"); - if (c2) { - *c2 = 0; - } - c2 = strstr(file->filetype, "stripped"); - if (c2) { - *c2 = 0; - } + if (file->stat.st_size < BSDIFFSIZE) { + /* thing is too small, always will be regenerated + * so no point in trying to figure out what kind + * of file it is + */ } else { - LOG(file, "Cannot find file type", "%s", filename); + file->filetype = getmagic(filename); } free(filename); - file->basename = strdup(basename(file->filename)); - file->dirname = strdup(dirname(file->filename)); + file->basename = g_path_get_basename(file->filename); + file->dirname = g_path_get_dirname(file->filename); } int file_sort_score(gconstpointer a, gconstpointer b) @@ -222,6 +251,10 @@ int file_sort_score(gconstpointer a, gconstpointer b) return 0; } +/* compare file to each deleted file. + * set file->rename_peer to best matched deleted file + * set file->rename_score to the score + */ static void score_file(GList *deleted_files, struct file *file) { GList *list2; @@ -246,61 +279,108 @@ static void score_file(GList *deleted_files, struct file *file) } } -void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list) +/* delete the first element of the list and return the new head */ +static GList* del_first(GList *list) { - GList *new_files = NULL; - GList *deleted_files = NULL; + /* The first list is the pointer to the list, the second is + * the pointer to what to delete */ + return g_list_delete_link(list, list); +} + + +/* Take a list, return a new list where the filter function returns true */ +static GList* new_filtered_list(GList *list, int version, int (*f)(struct file *file, int version)) +{ + /* make a list of new files, no peer */ + GList *newlist = NULL; + list = g_list_first(list); + for (; list; list = g_list_next(list)) { + struct file *file = list->data; + if (f(file, version)) { + newlist = g_list_prepend(newlist, file); + } + } + return newlist; +} + +static int renamed_file_p(struct file *file, int unused) +{ + return file->is_rename; +} +/* Return a new list of renamed files */ +static GList *new_list_renamed_files(GList *infiles) +{ + return new_filtered_list(infiles,0,renamed_file_p); +} + +/* Predicate that returns true if this is a new file in the stated version */ +static int new_file_p(struct file *file, int version) +{ + if ((file->last_change != version) || + (file->is_deleted) || + (!file->is_file) || + (file->peer)) { + return 0; + } + return 1; +} + +/* return a new list of the new files */ +static GList* list_new_files(struct manifest *manifest) +{ + GList *list = new_filtered_list(manifest->files, manifest->version, new_file_p); + /* call precompute_file_data for each file on list, return the list */ + GList *ret = list; + for (list = g_list_first(list); list ; list = g_list_next(list)) { + struct file *file=list->data; + precompute_file_data(manifest->version, manifest->component, file, true); + } + return ret; +} + +static int deleted_p(struct file *file, int version) +{ + if ((!file->is_deleted) || + (!file->peer) || + (file->last_change != version) || + (file->peer->is_dir || file->peer->is_link)) { + return 0; + } + return 1; +} + +static GList* list_deleted_files(struct manifest *manifest) +{ + GList *list = new_filtered_list(manifest->files, manifest->version, deleted_p); + GList *ret = list; + /* call precompute_file_data for each peer of file on list */ + for (list = g_list_first(list); list ; list = g_list_next(list)) { + struct file *file = list->data; + struct file *peer = file->peer; + /* Need to get things from the /full/ as we do not know + * which component may be coming from? */ + precompute_file_data(peer->last_change, "full", peer, false); + } + return ret; +} + +void rename_detection(struct manifest *manifest) +{ + GList *new_files; + GList *deleted_files; GList *list; struct file *file; - int old_rename = 0; - if (last_change != manifest->version) { - old_rename = 1; - } + new_files = list_new_files(manifest); - if (mcookie == NULL) { - mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS); - magic_load(mcookie, NULL); - } - - /* make a list of new files, no peer */ - list = g_list_first(manifest->files); - while (list) { - file = list->data; - list = g_list_next(list); - if ((file->last_change != manifest->version) || - (file->is_deleted) || - (!file->is_file) || - (file->peer)) { - continue; - } - - new_files = g_list_prepend(new_files, file); - precompute_file_data(manifest, file, old_rename, last_versions_list); - } - - /* if there are no new files, we're not having any renames -- early exit */ + /* no new files --> no renames -- early exit */ if (!new_files) { LOG(NULL, "No new files, no rename detection", "%s", manifest->component); return; } - /* make a list of newly deleted files that have a peer */ - list = g_list_first(manifest->files); - while (list) { - file = list->data; - list = g_list_next(list); - if ((!file->is_deleted) || - (!file->peer) || - (file->last_change != last_change) || - (file->peer->is_dir || file->peer->is_link)) { - continue; - } - - deleted_files = g_list_prepend(deleted_files, file); - precompute_file_data(NULL, file->peer, old_rename, last_versions_list); - } + deleted_files = list_deleted_files(manifest); /* nothing got deleted --> no renames --> early exit */ if (!deleted_files) { @@ -309,7 +389,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve return; } - /* for each new file, find the deleted file with the highest score, and store the score */ + /* for each new file, find the deleted file with the highest score, + * store it in file->rename_peer and store the score */ list = g_list_first(new_files); while (list) { file = list->data; @@ -317,20 +398,24 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve score_file(deleted_files, file); } +redo: /* sort all new files by score */ - + /* walk the sorted score list. + * pick the top score, + * check if the score is still valid, + * if not, recompute the score and resort + * This is probably an O(n^3). + */ new_files = g_list_sort(new_files, file_sort_score); - /* pick the top score, check if the score is still valid, if not, recompute the score and resort */ - - while (new_files) { + for (; new_files ; new_files = del_first(new_files)) { file = new_files->data; + if (file->rename_peer == NULL) { + continue; + } - if (file->rename_score < 15.0 || file->rename_peer == NULL) { - new_files = g_list_delete_link(new_files, new_files); - if (file->rename_peer) { - LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); - } + if (file->rename_score < 15.0) { + LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); continue; } @@ -338,10 +423,8 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve /* the candidate peer got already taken by another file! */ LOG(NULL, "Rename not done due to target already taken", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); file->rename_peer = NULL; - file->rename_score = -100; score_file(deleted_files, file); - new_files = g_list_sort(new_files, file_sort_score); - continue; + goto redo; } /* if valid and score is high enough, make the link by setting the flag and storing the hash */ @@ -349,13 +432,14 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve LOG(NULL, "Rename detected!", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); file->rename_peer->rename_peer = file; - /* must delete the file from the deleted list */ + /* must remove the file from the deleted list */ deleted_files = g_list_remove(deleted_files, file->rename_peer); hash_assign(file->hash, file->rename_peer->hash); file->is_rename = 1; file->rename_peer->is_rename = 1; - - new_files = g_list_delete_link(new_files, new_files); + if (!deleted_files) { + break; /* No more deleted files to rename */ + } } /* lather, rinse, repeat until all files have a target */ @@ -364,65 +448,48 @@ void rename_detection(struct manifest *manifest, int last_change, GList *last_ve g_list_free(deleted_files); } -static int file_found_in_older_manifest(struct manifest *from_manifest, struct file *searched_file) -{ - GList *list; - struct file *file; - list = g_list_first(from_manifest->files); - while (list) { - file = list->data; - list = g_list_next(list); - - if (file->is_deleted) { - continue; - } - if (!strcmp(file->filename, searched_file->filename)) { - return 1; - } - } - - return 0; -} - -void link_renames(GList *newfiles, struct manifest *from_manifest) +/* What do we need this for? + * + * rename_detection has already set up the links in the manifest it + * was given. + * + */ +void link_renames(GList *newfiles, int to_version) { GList *list1, *list2; GList *targets; struct file *file1, *file2; - targets = newfiles = g_list_sort(newfiles, file_sort_version); + targets = new_list_renamed_files(newfiles); + /* TODO: Check that g_list_sort is reasonable speed */ + targets = newfiles = g_list_sort(targets, file_sort_version); - list1 = g_list_first(newfiles); - - /* todo: sort newfiles and targets by hash */ - - while (list1) { + for (list1 = newfiles; list1; list1 = g_list_next(list1)) { file1 = list1->data; - list1 = g_list_next(list1); - if ((file1->peer || !file1->is_rename) || - (file1->is_deleted)) { + if (file1->peer || file1->is_deleted) { continue; } - /* now, file1 is the new file that got renamed. time to search the rename targets */ + /* now, file1 is the new file that got renamed. + * time to search the rename targets */ list2 = g_list_first(targets); - while (list2) { + for (; list2; list2 = g_list_next(list2)) { file2 = list2->data; - list2 = g_list_next(list2); - - if ((!file2->peer || !file2->is_rename) || + /* This is like deleted_p but not quite */ + /* deleted_p returns false for directories and links */ + if (!file2->peer || (!file2->is_deleted) || - (!file_found_in_older_manifest(from_manifest, file2))) { + (file2->last_change != to_version)) { continue; } if (hash_compare(file2->hash, file1->hash)) { file1->rename_peer = file2->peer; file1->peer = file2->peer; file2->peer->rename_peer = file1; - list2 = NULL; + break; } } } - free(from_manifest); + g_list_free(targets); }