commit 6c438ca96f5b92f7a993561a5e05da03616ce5d8 Author: Patrick McCarty Date: Wed Feb 24 10:00:31 2016 -0800 Initial commit Signed-off-by: Patrick McCarty diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d56a892 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +*~ +.deps +Makefile +Makefile.in +aclocal.m4 +autom4te.cache/ +compile +config.h +config.h.in +config.log +config.status +configure +depcomp +install-sh +*.o +.dirstamp +missing +stamp-h1 +DEADJOE +swupd_create_update +swupd_make_pack +swupd-update-server.log* +swupd_make_fullfiles +swupd-server-*.tar.gz +cscope.* +*.kdev4 diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..c23de84 --- /dev/null +++ b/COPYING @@ -0,0 +1,353 @@ +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 2 or later of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + + + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..2af87ed --- /dev/null +++ b/Makefile.am @@ -0,0 +1,110 @@ +EXTRA_DIST = COPYING + +AM_CFLAGS = -fPIC -Iinclude/ -O2 -g -Wall -W -Wformat-security -D_FORTIFY_SOURCE=2 -DSWUPD_LINUX_ROOTFS -fno-common -std=gnu99 + +bin_PROGRAMS = \ + swupd_create_update \ + swupd_make_pack \ + swupd_make_fullfiles + +# TODO: +# check_PROGRAMS = +# ...add test/* binaries as appopropriate + +swupd_create_update_SOURCES = \ + src/analyze_fs.c \ + src/chroot.c \ + src/config.c \ + src/delta.c \ + src/fullfiles.c \ + src/globals.c \ + src/groups.c \ + src/helpers.c \ + src/heuristics.c \ + src/log.c \ + src/main.c \ + src/manifest.c \ + src/pack.c \ + src/rename.c \ + src/signature.c \ + src/stats.c \ + src/type_change.c \ + src/versions.c \ + src/xattrs.c + +swupd_make_pack_SOURCES = \ + src/analyze_fs.c \ + src/config.c \ + src/delta.c \ + src/globals.c \ + src/helpers.c \ + src/log.c \ + src/make_packs.c \ + src/manifest.c \ + src/pack.c \ + src/rename.c \ + src/signature.c \ + src/stats.c \ + src/xattrs.c + +swupd_make_fullfiles_SOURCES = \ + src/analyze_fs.c \ + src/config.c \ + src/delta.c \ + src/fullfiles.c \ + src/globals.c \ + src/helpers.c \ + src/log.c \ + src/make_fullfiles.c \ + src/manifest.c \ + src/pack.c \ + src/rename.c \ + src/signature.c \ + src/stats.c \ + src/xattrs.c + +AM_CPPFLAGS = $(glib_CFLAGS) -I$(top_srcdir)/include + +swupd_create_update_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ + $(bsdiff_LIBS) + +swupd_make_pack_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ + $(bsdiff_LIBS) + +swupd_make_fullfiles_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ + $(bsdiff_LIBS) + +if ENABLE_LZMA +swupd_create_update_LDADD += \ + $(lzma_LIBS) +swupd_make_pack_LDADD += \ + $(lzma_LIBS) +swupd_make_fullfiles_LDADD += \ + $(lzma_LIBS) +endif + +noinst_HEADERS = \ + include/swupd.h \ + include/xattrs.h + +distclean-local: + rm -rf aclocal.m4 autogen.sh autom4te.cache config.h.in config.h.in~ configure depcomp install-sh Makefile.in missing compile + +release: + @git rev-parse v$(PACKAGE_VERSION) &> /dev/null; \ + if [ "$$?" -eq 0 ]; then \ + echo "Error: Release for $(PACKAGE_VERSION) already exists."; \ + echo "Bump version in configure.ac before releasing."; \ + exit 1; \ + fi + @git tag -a -m "$(PACKAGE_NAME) release $(PACKAGE_VERSION)" v$(PACKAGE_VERSION) + @printf "\nNew release $(PACKAGE_VERSION) tagged!\n\n" diff --git a/README b/README new file mode 100644 index 0000000..9669a30 --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +The swupd-server package provides a reference implementation of a software +update server-side component that generates update content consumable by a +software update client (swupd-client). Such content includes manifests that +describe incremental changes in the OS from build to build, binary deltas, +full copies of files (fullfiles) that were added/changed from a previous +build, and packs composed of binary deltas and/or fullfiles. diff --git a/basic_creator.sh b/basic_creator.sh new file mode 100755 index 0000000..a341792 --- /dev/null +++ b/basic_creator.sh @@ -0,0 +1,65 @@ +#!/bin/bash +VER=$1 +XZ_DEFAULTS="--threads 0" +PREVREL=`cat /var/lib/update/image/latest.version` + +SWUPDREPO=/usr/src/clear-projects/swupd-server +BUNDLEREPO=/usr/src/clear-projects/clr-bundles +UPDATEDIR=/var/lib/update + +error() { + echo "${1:-"Unknown Error"}" + exit 1 +} + +# remove things from $BUNDLEREPO/bundles to make local build faster, +# eg: all of openstack, pnp, bat, cloud stuff, non-basic scripting language bundles +${SWUPDREPO}/mk_groups_ini.sh + +export SWUPD_CERTS_DIR="/root/swupd-certs" +export LEAF_KEY="leaf.key.pem" +export LEAF_CERT="leaf.cert.pem" +export CA_CHAIN_CERT="ca-chain.cert.pem" +export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase" + +${SWUPDREPO}/swupd_create_update --osversion ${VER} +${SWUPDREPO}/swupd_make_fullfiles ${VER} + +pushd ${SWUPDREPO} + +# create zero packs +MOM=${UPDATEDIR}/www/${VER}/Manifest.MoM +if [ ! -e ${MOM} ]; then + error "no ${MOM}" +fi +BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }') +for BUNDLE in $BUNDLE_LIST; do + #background them for parallelization + ${SWUPDREPO}/swupd_make_pack 0 ${VER} ${BUNDLE} & +done +#now await all completing +for job in $(jobs -p); do + wait ${job} + RET=$? + if [ "$RET" != "0" ]; then + error "zero pack subprocessor failed" + fi +done + +# create delta packs for 2 versions back +NUM_PACKS=2 +${SWUPDREPO}/pack_maker.sh ${VER} ${NUM_PACKS} + +popd ${SWUPDREPO} + +# expose the new build to staging / testing +echo ${VER} > ${UPDATEDIR}/image/latest.version +STAGING=$(cat ${UPDATEDIR}/www//version/formatstaging/latest) +if [ "${STAGING}" -lt "${VER}" ]; then + echo ${VER} > ${UPDATEDIR}/www//version/formatstaging/latest +fi + +#valgrind /usr/src/clear-projects/swupd-server/swupd_create_update ${VER} +time hardlink /var/lib/update/image/${VER}/* +time hardlink /var/lib/update/image/$PREVREL /var/lib/update/image/${VER} +time hardlink /var/lib/update/www/$PREVREL /var/lib/update/www/${VER} diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..97cc41b --- /dev/null +++ b/configure.ac @@ -0,0 +1,45 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ([2.66]) +AC_INIT(swupd-server, 3.0.0, timothy.c.pepper@linux.intel.com) +AM_INIT_AUTOMAKE([foreign -Wall -W subdir-objects]) +AM_SILENT_RULES([yes]) +AC_PROG_CC +AM_PROG_CC_C_O +AC_LANG(C) +AC_CONFIG_HEADERS([config.h]) +PKG_CHECK_MODULES([bsdiff], [bsdiff]) +PKG_CHECK_MODULES([glib], [glib-2.0, gthread-2.0]) +PKG_CHECK_MODULES([zlib], [zlib]) +PKG_CHECK_MODULES([openssl], [libcrypto >= 0.9.8]) +AC_CHECK_LIB([magic], [magic_open], [], [AC_MSG_ERROR([the magic library is missing])]) +AC_CHECK_PROGS(TAR, tar) + +AC_ARG_ENABLE([bzip2], + [AS_HELP_STRING([--disable-bzip2],[Do not use bzip2 compression (uses bzip2 by default)])]) + +AC_ARG_ENABLE([lzma], + [AS_HELP_STRING([--disable-lzma],[Do not use lzma compression (uses lzma by default)])]) +AC_ARG_ENABLE( + [stateless], + AS_HELP_STRING([--disable-stateless],[OS is not stateless, do not ignore configuration files (stateless by default)]), + AC_DEFINE(SWUPD_WITH_STATELESS,0,[OS is not stateless]), + AC_DEFINE(SWUPD_WITH_STATELESS,1,[OS is stateless]) +) + +AS_IF([test "$enable_bzip2" != "no"], [ + AC_CHECK_LIB([bz2], [BZ2_bzBuffToBuffCompress], [], [AC_MSG_ERROR([the libbz2 library is missing])]) + AC_CHECK_PROGS(BZIP2, bzip2) + AC_DEFINE(SWUPD_WITH_BZIP2,1,[Use bzip2 compression]) +]) + +AS_IF([test "$enable_lzma" != "no"], [ + PKG_CHECK_MODULES([lzma], [liblzma]) + AC_CHECK_PROGS(XZ, xz) + AC_DEFINE(SWUPD_WITH_LZMA,1,[Use lzma compression]) +]) +AM_CONDITIONAL([ENABLE_LZMA], [test "$enable_lzma" != "no"]) + +AC_CONFIG_FILES([Makefile]) +AC_OUTPUT diff --git a/include/swupd.h b/include/swupd.h new file mode 100644 index 0000000..897c8dd --- /dev/null +++ b/include/swupd.h @@ -0,0 +1,241 @@ +#ifndef __INCLUDE_GUARD_SWUPD_H +#define __INCLUDE_GUARD_SWUPD_H + +#include +#include +#include +#include + +#include "config.h" + +#define __unused__ __attribute__ ((__unused__)) + +// SWUPD_NUM_PACKS is also "PREV_CHECK" in releas tool swupd_bb.py (change both) +#define SWUPD_NUM_PACKS 4 +#define SWUPD_NUM_MANIFEST_DELTAS 25 +#define SWUPD_DEFAULT_FORMAT 3 + +#define SWUPD_SERVER_STATE_DIR "/var/lib/update" + +#if SWUPD_WITH_SELINUX +#define TAR_PERM_ATTR_ARGS "--preserve-permissions --xattrs --xattrs-include='*' --selinux" +#else +#define TAR_PERM_ATTR_ARGS "--preserve-permissions --xattrs --xattrs-include='*'" +#endif + +#if SWUPD_WITH_STATELESS +#define OS_IS_STATELESS 1 +#else +#define OS_IS_STATELESS 0 +#endif + +/* Build types */ +#define REGULAR_BUILD 0 +#define FIX_BUILD 1 +#define DEV_BUILD 2 + +/* We don't have BZIP2 support in Android. This define translates to "ifdef 0" + * and demarcates codes which otherwise would enable BZIP2. If this define + * is changed, the Android update client will break. DO NOT CHANGE THIS + * DEFINE IN AN ANDROID TARGETED BUILD WITHOUT ENABLING/TESTING BZIP2 SUPPORT + * IN THE ANDROID CLIENT OS BUILD. + */ +#ifdef SWUPD_WITH_BZIP2 +#include +#endif + +/* Build toggle for lzma support. + */ +#ifdef SWUPD_WITH_LZMA +#include +#endif + +struct manifest { + int version; + int prevversion; + char *component; + int count; + uint64_t contentsize; + GList *files; + GList *manifests; /* as struct file */ + + GList *submanifests; /* as struct manifest */ +}; + +struct file; +struct header; + +struct update_stat { + uint64_t st_mode; + uint64_t st_uid; + uint64_t st_gid; + uint64_t st_rdev; + uint64_t st_size; +}; + +#define DIGEST_LEN_SHA256 64 +/* +1 for null termination */ +#define SWUPD_HASH_LEN (DIGEST_LEN_SHA256+1) + +struct file { + char *filename; + char hash[SWUPD_HASH_LEN]; + bool use_xattrs; + int last_change; + + struct update_stat stat; + + /* the following are exclusive */ + unsigned int is_dir : 1; + unsigned int is_file : 1; + unsigned int is_link : 1; + unsigned int is_deleted : 1; + unsigned int is_manifest : 1; + + /* and these are modifiers */ + unsigned int is_config : 1; + unsigned int is_state : 1; + unsigned int is_boot : 1; + unsigned int is_rename : 1; + + struct file *peer; /* same file in another manifest */ + + /* data fields to help rename detection */ + double rename_score; + struct file *rename_peer; + char *alpha_only_filename; /* filename minus all numerics/etc */ + char *filetype; + char *basename; + char *dirname; + /* end of rename detection fields */ + + unsigned int multithread : 1; /* if set to 1, current file is computed in a + multithreaded process for fullfile creation */ +}; + +struct packdata { + char *module; + int from; + int to; + int fullcount; + struct manifest *end_manifest; +}; + +extern int current_version; +extern int newversion; +extern int minversion; +extern char *format_string; +extern bool enable_signing; + +extern char *state_dir; +extern char *packstage_dir; +extern char *image_dir; +extern char *staging_dir; + +extern bool init_globals(void); +extern void free_globals(void); +extern bool set_format_string(char *); +extern void check_root(void); +extern bool set_state_dir(char *); +extern bool init_state_globals(void); +extern void free_state_globals(void); + +extern int file_sort_hash(gconstpointer a, gconstpointer b); +extern int file_sort_filename(gconstpointer a, gconstpointer b); +extern int file_sort_version(gconstpointer a, gconstpointer b); + +extern bool read_configuration_file(char *filename); +extern void release_configuration_data(void); +extern char * config_image_base(void); +extern char * config_output_dir(void); +extern char * config_empty_dir(void); +extern int config_initial_version(void); + +extern void read_current_version(char *filename); +extern void write_new_version(char *filename, int version); +extern void write_cookiecrumbs_to_download_area(int version); +extern void ensure_version_image_exists(int version); +extern GList *get_last_versions_list(int next_version, int max_versions); + +extern char *file_type_to_string(struct file *file); +extern struct manifest *manifest_from_file(int version, char *module); +extern void free_manifest(struct manifest *manifest); +extern struct manifest *alloc_manifest(int version, char *module); +extern int match_manifests(struct manifest *m1, struct manifest *m2); +extern void sort_manifest_by_version(struct manifest *manifest); +extern int prune_manifest(struct manifest *manifest); +extern void create_manifest_delta(int oldversion, int newversion, char *module); +extern void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list); +extern void subtract_manifests(struct manifest *m1, struct manifest *m2); +extern void nest_manifest(struct manifest *parent, struct manifest *sub); +extern void nest_manifest_file(struct manifest *parent, struct file *file); +extern int manifest_subversion(struct manifest *parent, char *group); + +extern void account_new_file(void); +extern void account_deleted_file(void); +extern void account_changed_file(void); +extern void print_statistics(int version1, int version2); + +extern struct manifest *full_manifest_from_directory(int version); +extern struct manifest *sub_manifest_from_directory(char *component, int version); +extern void add_component_hashes_to_manifest(struct manifest *compm, struct manifest *fullm); +extern int write_manifest(struct manifest *manifest); + +extern void hash_assign(char *src, char *dest); +extern bool hash_compare(char *hash1, char *hash2); +extern void hash_set_zeros(char *hash); +extern bool hash_is_zeros(char *hash); +extern int compute_hash(struct file *file, char *filename) __attribute__((warn_unused_result)); + +extern void prepare_delta_dir(struct manifest *manifest); +extern void create_fullfiles(struct manifest *manifest); +extern bool create_download_content_for_group(const char *group); +extern bool compute_hash_with_xattrs(const char *group); + +extern void clean_package_caches(char *subdir); +extern void update_chroot(char *dir); +extern void chroot_create_full(int newversion); + +extern void read_group_file(char *filename); +extern void release_group_file(void); +extern char *group_groups(char *group); +extern char *group_packages(char *group); +extern char *next_group(void); + +extern void apply_heuristics(struct manifest *manifest); +extern int make_pack(struct packdata *pack); + +extern void maximize_to_full(struct manifest *MoM, struct manifest *full); +extern void recurse_manifest(struct manifest *manifest); +extern void consolidate_submanifests(struct manifest *manifest); +extern void populate_file_struct(struct file *file, char *filename); +extern void download_exta_base_content(void); + +extern char *get_elapsed_time(struct timeval *t1, struct timeval *t2); +extern void init_log(int version); +extern void init_log_stdout(void); +extern void close_log(int version, int exit_status); +extern void __log_message(struct file *file, char *msg, char *filename, int linenr, const char *fmt, ...); +#define LOG(file, msg, fmt...) __log_message(file, msg, __FILE__, __LINE__, fmt) +extern int previous_version_manifest(struct manifest *mom, char *name); + +extern void type_change_detection(struct manifest *manifest); + +extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); +extern void link_renames(GList *newfiles, struct manifest *from_manifest); +extern void __create_delta(struct file *file, int from_version); + +extern void account_delta_hit(void); +extern void account_delta_miss(void); + +extern FILE * fopen_exclusive(const char *filename); /* no mode, opens for write only */ +extern void dump_file_info(struct file *file); +extern void string_or_die(char **strp, const char *fmt, ...); +extern void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time); +extern int system_argv(char *const argv[]); + +extern bool signature_initialize(void); +extern void signature_terminate(void); +extern bool signature_sign(const char *filename); + +#endif diff --git a/include/xattrs.h b/include/xattrs.h new file mode 100644 index 0000000..6402a28 --- /dev/null +++ b/include/xattrs.h @@ -0,0 +1,42 @@ +#ifndef __INCLUDE_GUARD_XATTRS_H +#define __INCLUDE_GUARD_XATTRS_H + +#include + +/* + * Best effort to copy the extended attributes from one file to another. + * + * @param src_filename - The source file from which the extended attributes + * will be read. + * @param dst_filename - The destination file where the source file extended + * attributes have to be copied. + * @return - None. + */ +void xattrs_copy(const char *src_filename, const char *dst_filename); + +/* + * Attempt to pack into a data blob the extended attributes of the given file. + * The data blob will be allocated and possibly filled with first the list + * of all the extended attributes names, and then their values. + * @param filename - The file from which the extended attributes will be + * read and packed into the data blob. + * @param blob - The data blob pointer into which the extended attributes will + * be packed. (the data blob content has to be freed by the caller). + * @param blob_len - The returned data blob length. + * @return - None. + */ +void xattrs_get_blob(const char *filename, char **blob, size_t *blob_len); + +/* + * Compare the extended attributes from one file to another. + * + * @param filename1 - The first file used for the extended attributes + * comparison. + * @param filename2 - The second file used for the extended attributes + * comparison. + * @return - 0 if the extended attributes are identical, -1 if they are + * different. + */ +int xattrs_compare(const char *filename1, const char *filename2); + +#endif /* __INCLUDE_GUARD_XATTRS_H */ diff --git a/mk_groups_ini.sh b/mk_groups_ini.sh new file mode 100755 index 0000000..aa6a948 --- /dev/null +++ b/mk_groups_ini.sh @@ -0,0 +1,32 @@ +#! /bin/bash + +COMMON=${COMMON:-"/root/common"} +BUNDLEREPO=${BUNDLEREPO:-"/root/clr-bundles"} + +SWUPD_SERVER_DIR=${SWUPD_SERVER_DIR:-"/var/lib/update"} +SWUPD_GROUPS_INI=${SWUPD_GROUPS_INI:-"$SWUPD_SERVER_DIR/groups.ini"} +SWUPD_SERVER_INI=${SWUPD_SERVER_INI:-"$SWUPD_SERVER_DIR/server.ini"} + +if [ ! -d "$SWUPD_SERVER_DIR" ]; then + mkdir -p $SWUPD_SERVER_DIR/{image,www} +fi + +if [ ! -f "$SWUPD_SERVER_INI" ]; then + template=$(echo $BASH_SOURCE) + template=$(dirname $template) + cp -p $template/server.ini $SWUPD_SERVER_INI +fi + +if [ ! -f "$SWUPD_SERVER_DIR/image/latest.version" ]; then + echo "0" > $SWUPD_SERVER_DIR/image/latest.version +fi + +echo "rebuilding $SWUPD_GROUPS_INI based on $BUNDLEREPO" +rm -f $SWUPD_GROUPS_INI + +for bundle in $(ls $BUNDLEREPO/bundles) +do + echo "[$bundle]" >> $SWUPD_GROUPS_INI + echo "group=$bundle" >> $SWUPD_GROUPS_INI + echo "" >> $SWUPD_GROUPS_INI +done diff --git a/pack_maker.sh b/pack_maker.sh new file mode 100755 index 0000000..59604a5 --- /dev/null +++ b/pack_maker.sh @@ -0,0 +1,111 @@ +#!/bin/bash -e +# usage: +# pack_maker.sh +# pack_maker.sh 2820 2 +# pack_maker.sh 2730 3 +# +# For each bundle changed as of the Manifest.MoM at the , +# create version-pair packs going back +# versions of that bundle. For example if bundle "editors" changed in 2650 +# and the script is run as "pack_maker.sh 2650 2", it will find that +# "editors" last two change points were 2550 and 2260 and thus create a +# 2550-to-2650 and a 2260-to-2650 version pair pack for the "editors" +# bundle. + +#NOTE: fullfiles must be created before calling this script +#NOTE: zero packs need similarly created + +#NOTE: build numbers are sparse, search back for the next oldest existing build +#NOTE: in the future isn't simply -10, and when that change happens we must also +# consider which version-pair packs are meaningful for intermediate builds +# and revert builds. + +SWUPDREPO=${SWUPDREPO:-"/root/swupd-server"} +BUNDLEREPO=${BUNDLEREPO:-"/root/clr-bundles"} +SWUPDWEBDIR="/var/lib/update/www" +SWUPD_CERTS_DIR=${SWUPD_CERTS_DIR=:-"/root/swupd-certs"} + +export XZ_DEFAULTS="--threads 0" + +export SWUPD_CERTS_DIR +export LEAF_KEY="leaf.key.pem" +export LEAF_CERT="leaf.cert.pem" +export CA_CHAIN_CERT="ca-chain.cert.pem" +export PASSPHRASE="${SWUPD_CERTS_DIR}/passphrase" + +VER=$1 +BACK_COUNT=$2 + +if [ "${BACK_COUNT}" == "" ]; then + echo "missing to pack count" + exit 1 +fi + +if [ "${VER}" == "" ]; then + echo "missing to version" + exit 1 +fi + +MOM=${SWUPDWEBDIR}/${VER}/Manifest.MoM +if [ ! -e ${MOM} ]; then + echo "invalid version (no ${MOM})" + exit 1 +fi + +#time ${SWUPDREPO}/swupd_make_fullfiles ${VER} + +BUNDLE_LIST=$(cat ${MOM} | awk -v V=${VER} '$1 ~ /^M\./ && $3 == V { print $4 }') + +# build packs for all bundles changed in $VER +for BUNDLE in $BUNDLE_LIST; do + #${SWUPDREPO}/swupd_make_pack 0 ${VER} ${BUNDLE} & + + BUNDLE_VER_LIST="" + MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$VER/Manifest.MoM | grep "^previous:" | cut -f 2) + BUNDLE_VER="" + COUNT=0 + while [ $COUNT -lt $BACK_COUNT ] && [ $MANIFEST_VER -gt 0 ]; do + MOM=${SWUPDWEBDIR}/$MANIFEST_VER/Manifest.MoM + if [ -e ${MOM} ]; then + BUNDLE_VER=$(cat ${MOM} | grep " ${BUNDLE}$" | cut -f 3) + if [ "$BUNDLE_VER" != "" ]; then + BUNDLE_VER_LIST="$BUNDLE_VER $BUNDLE_VER_LIST" + MANIFEST_VER=$(cat ${SWUPDWEBDIR}/$BUNDLE_VER/Manifest.MoM | grep "^previous:" | cut -f 2) + let COUNT=$((COUNT+1)) + else + # back in history to where bundle didn't exist yet + MANIFEST_VER=0 + fi + else + # seek backwards until a manifest if found + MANIFEST_VER=$((MANIFEST_VER-10)) + fi + done + + for v in $BUNDLE_VER_LIST; do + if [ -e /var/lib/update/www/${VER}/pack-${BUNDLE}-from-$v.tar ]; then + echo "${VER}/pack-${BUNDLE}-from-$v.tar already exists, skipping." + else + ${SWUPDREPO}/swupd_make_pack $v ${VER} ${BUNDLE} + fi + done +done + +for job in $(jobs -p) +do + wait ${job} + RET=$? + if [ "$RET" != "0" ]; then + echo "pack creation failure!" + exit $RET + fi +done + +# NOTE: your devops will want to expose the completed swupd server build to +# trial usage at some point. This would be done via code like: +# +# echo ${VER} > ${SWUPDWEBDIR}/../image/latest.version +# STAGING=$(cat ${SWUPDWEBDIR}/version/formatstaging/latest) +# if [ "${STAGING}" -lt "${VER}" ]; then +# echo ${VER} > ${SWUPDWEBDIR}/version/formatstaging/latest +# fi diff --git a/packfsck.pl b/packfsck.pl new file mode 100755 index 0000000..d7171db --- /dev/null +++ b/packfsck.pl @@ -0,0 +1,82 @@ +#!/usr/bin/perl -w +# +# +# arguments: +# perl packfsck.pl + + +my $target = $ARGV[0]; +system("rm /tmp/Manifest"); +system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/Manifest https://download.clearlinux.org/update/$target/Manifest.os-core"); + + +my $from = $target; +while ($from > $target - 100) { + $from = $from - 10; + print "Testing the $from-$target pack\n"; + system("rm /tmp/pack.tar"); + system("wget --quiet --no-proxy --no-check-certificate --output-document=/tmp/pack.tar https://download.clearlinux.org/update/$target/pack-os-core-from-$from.tar"); + + + + open FILE, ") { + my $line = $_; + + + if ($line =~ /(^[FDLr\.]+)\s*([0-9a-f]+)\s*([0-9]+)\s*+(.*)\n/) { + my $type = $1; + my $hash = $2; + my $version = $3; + my $filename = $4; + if ($version > $from) { + $expected_hashes{"$hash"} = 1; + $files{"$hash"} = $filename; +# print "$1 - $2 - $3 - $4\n"; + } + } + } + close FILE; + + + open FILE2, "-|", "tar -tf /tmp/pack.tar 2> /dev/null"; + + while () { + my $line = $_; + + + if ($line =~ /^\.\/delta\/[0-9]+.[0-9]+.([0-9a-f]+)/) { + my $hash = $1; + if (!defined($expected_hashes{"$hash"})) { + print "\tUnexpected delta hash found $hash\n"; + } + $expected_hashes{"$hash"} = 2; + } + if ($line =~ /\.\/staged\/([0-9a-f]+)/) { + my $hash = $1; + if (!defined($expected_hashes{"$hash"})) { + print "\tUnexpected staged hash found $hash\n"; + } + $expected_hashes{"$hash"} = 2; + } + } + close FILE2; + + my $count = 0; + foreach my $key ( keys %expected_hashes ) + { + my $value = $expected_hashes{$key}; + if ($value eq 1) { + my $fn = $files{$key}; + print "\tFile $fn ($key) is not in the pack\n"; + $count = $count = 1; + } + } + if ($count < 1) { + print "\tThe $from-$target pack has no files missing\n"; + } +} diff --git a/publish.sh b/publish.sh new file mode 100755 index 0000000..838c233 --- /dev/null +++ b/publish.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Update content is created and output into the www directory, but the +# published "latest" version number is only bumped in +# www/version/formatstaging/latest. This enables QA on devices whose update +# client is set to query the www/version/formatstaging/latest. Once that +# QA is complete, this publish script is run to bump the production +# format's "latest" version number. + +if [[ $# < 1 ]]; then + echo "First argument must be a format number" + echo "$0 aborting" + exit 1 +fi +FORMAT=$1 + +ver1=`cat /var/lib/update/www/version/formatstaging/latest` +ver2=`cat /var/lib/update/www/version/format$FORMAT/latest` +cp /var/lib/update/www/version/formatstaging/latest /var/lib/update/www/version/format$FORMAT/latest +echo "Updated from $ver2 to $ver1 in format $FORMAT" diff --git a/server.ini b/server.ini new file mode 100644 index 0000000..f933833 --- /dev/null +++ b/server.ini @@ -0,0 +1,4 @@ +[Server] +emptydir=/var/lib/update/empty/ +imagebase=/var/lib/update/image/ +outputdir=/var/lib/update/www/ diff --git a/src/analyze_fs.c b/src/analyze_fs.c new file mode 100644 index 0000000..679b8e3 --- /dev/null +++ b/src/analyze_fs.c @@ -0,0 +1,464 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * cguiraud + * Tim Pepper + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" +#include "xattrs.h" + +static GThreadPool *threadpool; + +void hash_assign(char *src, char *dst) +{ + memcpy(dst, src, SWUPD_HASH_LEN-1); + dst[SWUPD_HASH_LEN-1] = '\0'; +} + +bool hash_compare(char *hash1, char *hash2) +{ + if (bcmp(hash1, hash2, SWUPD_HASH_LEN-1) == 0) { + return true; + } else { + return false; + } +} + +bool hash_is_zeros(char *hash) +{ + return hash_compare("0000000000000000000000000000000000000000000000000000000000000000", hash); +} + +void hash_set_zeros(char *hash) +{ + hash_assign("0000000000000000000000000000000000000000000000000000000000000000", hash); +} + +static void hmac_sha256_for_data(char *hash, + const unsigned char *key, size_t key_len, + const unsigned char *data, size_t data_len) +{ + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int digest_len = 0; + char *digest_str; + unsigned int i; + + if (data == NULL) { + hash_set_zeros(hash); + return; + } + + if (HMAC(EVP_sha256(), (const void *)key, key_len, data, data_len, digest, &digest_len) == NULL) { + hash_set_zeros(hash); + return; + } + + digest_str = calloc((digest_len * 2) + 1, sizeof(char)); + if (digest_str == NULL) { + assert(0); + } + + for (i = 0; i < digest_len; i++) { + sprintf(&digest_str[i * 2], "%02x", (unsigned int)digest[i]); + } + + hash_assign(digest_str, hash); + free(digest_str); +} + +static void hmac_sha256_for_string(char *hash, + const unsigned char *key, size_t key_len, + const char *str) +{ + if (str == NULL) { + hash_set_zeros(hash); + return; + } + + hmac_sha256_for_data(hash, key, key_len, (const unsigned char *)str, strlen(str)); +} + + +static void hmac_compute_key(const char *filename, + const struct update_stat *updt_stat, + char *key, size_t *key_len, bool use_xattrs) +{ + char *xattrs_blob = (void *)0xdeadcafe; + size_t xattrs_blob_len = 0; + + if (use_xattrs) { + xattrs_get_blob(filename, &xattrs_blob, &xattrs_blob_len); + } + + hmac_sha256_for_data(key, (const unsigned char *)updt_stat, + sizeof(struct update_stat), + (const unsigned char *)xattrs_blob, + xattrs_blob_len); + + if (hash_is_zeros(key)) { + *key_len = 0; + } else { + *key_len = SWUPD_HASH_LEN-1; + } + + if (xattrs_blob_len != 0) { + free(xattrs_blob); + } +} + +void populate_file_struct(struct file *file, char *filename) +{ + struct stat stat; + int ret; + + memset(&stat, 0, sizeof(stat)); + ret = lstat(filename, &stat); + if (ret < 0) { + LOG(NULL, "stat error ", "%s: %s", filename, strerror(errno)); + file->is_deleted = 1; + return; + } + file->stat.st_mode = stat.st_mode; + file->stat.st_uid = stat.st_uid; + file->stat.st_gid = stat.st_gid; + file->stat.st_rdev = stat.st_rdev; + file->stat.st_size = stat.st_size; + + if (S_ISLNK(stat.st_mode)) { + file->is_file = 0; + file->is_dir = 0; + file->is_link = 1; + memset(&file->stat.st_mode, 0, sizeof(file->stat.st_mode)); + return; + } + + if (S_ISDIR(stat.st_mode)) { + file->is_file = 0; + file->is_dir = 1; + file->is_link = 0; + file->stat.st_size = 0; + return; + } + + /* if we get here, this is a regular file */ + file->is_file = 1; + file->is_dir = 0; + file->is_link = 0; + return; +} + +/* this function MUST be kept in sync with the client + * return is -1 if there was an error. If the file does not exist, + * a "0000000..." hash is returned as is our convention in the manifest + * for deleted files. Otherwise file->hash is set to a non-zero hash. */ +int compute_hash(struct file *file, char *filename) +{ + int ret; + char key[SWUPD_HASH_LEN]; + size_t key_len; + unsigned char *blob; + FILE *fl; + + if (file->is_deleted) { + hash_set_zeros(file->hash); + return 0; + } + + hash_set_zeros(key); + + if (file->is_link) { + char link[PATH_MAX]; + memset(link, 0, PATH_MAX); + + ret = readlink(filename, link, PATH_MAX-1); + + if (ret >= 0) { + hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs); + hmac_sha256_for_string(file->hash, + (const unsigned char *)key, + key_len, + link); + return 0; + } else { + LOG(NULL, "readlink error ", "%i - %i / %s", ret, errno, strerror(errno)); + return -1; + } + } + + if (file->is_dir) { + hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs); + hmac_sha256_for_string(file->hash, + (const unsigned char *)key, + key_len, + file->filename); //file->filename not filename + return 0; + } + + /* if we get here, this is a regular file */ + fl = fopen(filename, "r"); + if (!fl) { + LOG(NULL, "file open error ", "%s: %s", filename, strerror(errno)); + return -1; + } + blob = mmap(NULL, file->stat.st_size, PROT_READ, MAP_PRIVATE, fileno(fl), 0); + assert(!(blob == MAP_FAILED && file->stat.st_size != 0)); + + hmac_compute_key(filename, &file->stat, key, &key_len, file->use_xattrs); + hmac_sha256_for_data(file->hash, + (const unsigned char *)key, + key_len, + blob, + file->stat.st_size); + munmap(blob, file->stat.st_size); + fclose(fl); + return 0; +} + +static void get_hash(gpointer data, gpointer user_data) +{ + struct file *file = data; + char *base = user_data; + char *filename = NULL; + int ret; + + if (base == NULL) { + string_or_die(&filename, "%s", file->filename); + } else { + string_or_die(&filename, "%s/%s", base, file->filename); + } + assert(filename); + + file->use_xattrs = compute_hash_with_xattrs(filename); + + populate_file_struct(file, filename); + ret = compute_hash(file, filename); + if (ret != 0) { + printf("Hash computation failed\n"); + assert(0); + } + + free(filename); +} + +/* disallow characters which can do unexpected things when the filename is + * used on a tar command line via system("tar [args] filename [more args]"); + */ +static bool illegal_characters(char *filename) +{ + char c; + int i; +#define BAD_CHAR_COUNT 11 + char bad_chars[BAD_CHAR_COUNT] = {';', '&', '|', '*', '`', '/', + '<', '>', '\\', '\"', '\''}; + + // these breaks the tar transform sed-like expression, + // hopefully can remove this check after moving to libtar + if (strncmp(filename, "+", 1) == 0) { + printf("WARNING: Filename %s begins with '+'! ...skipping.\n", filename); + return true; + } + if (strstr(filename, "+package+") != NULL) { + printf("WARNING: Filename %s contains \"+package+\"! ...skipping.\n", filename); + return true; + } + + for(i=0; id_name, ".") == 0) || + (strcmp(entry->d_name, "..") == 0) || + (illegal_characters(entry->d_name))) { + continue; + } + + file = calloc(1, sizeof(struct file)); + if (!file) { + break; + } + + file->last_change = manifest->version; + string_or_die(&file->filename, "%s/%s", subpath, entry->d_name); + + string_or_die(&fullname, "%s/%s", fullpath, entry->d_name); + populate_file_struct(file, fullname); + free(fullname); + + if (entry->d_type == DT_DIR) { + iterate_directory(manifest, pathprefix, file->filename, do_hash); + } + + /* if for some reason there is a file in the official build + * which should not be included in the Manifest, then open a bug + * to get it removed, and work around its presence by + * excluding it here, eg: + if (strncmp(file->filename, "/dev/", 5) == 0) { + continue; + } + */ + + if (do_hash) { + /* compute the hash from a thread */ + ret = g_thread_pool_push(threadpool, file, &err); + if (ret == FALSE){ + printf("GThread hash computation push error\n"); + printf("%s\n", err->message); + closedir(dir); + return; + } + } + manifest->files = g_list_prepend(manifest->files, file); + manifest->count++; + } + closedir(dir); + free(fullpath); +} + + +struct manifest *full_manifest_from_directory(int version) +{ + struct manifest *manifest; + char *dir; + + LOG(NULL, "Computing hashes", "for %i/full", version); + + manifest = alloc_manifest(version, "full"); + + string_or_die(&dir, "%s/%i/full", image_dir, version); + + threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL); + + iterate_directory(manifest, dir, "", true); + + /* wait for the hash computation to finish */ + g_thread_pool_free(threadpool, FALSE, TRUE); + free(dir); + + manifest->files = g_list_sort(manifest->files, file_sort_filename); + + return manifest; +} + +struct manifest *sub_manifest_from_directory(char *component, int version) +{ + struct manifest *manifest; + char *dir; + + LOG(NULL, "Creating component manifest", "for %i/%s", version, component); + + manifest = alloc_manifest(version, component); + + string_or_die(&dir, "%s/%i/%s", image_dir, version, component); + + iterate_directory(manifest, dir, "", false); + + free(dir); + + manifest->files = g_list_sort(manifest->files, file_sort_filename); + + return manifest; +} + +/* get hashes out of full manifest and add them into the component manifest */ +void add_component_hashes_to_manifest(struct manifest *compm, struct manifest *fullm) +{ + GList *list1, *list2; + struct file *file1, *file2; + int ret; + + assert(compm); + assert(fullm); + + compm->files = g_list_sort(compm->files, file_sort_filename); + fullm->files = g_list_sort(fullm->files, file_sort_filename); + + list1 = g_list_first(compm->files); + list2 = g_list_first(fullm->files); + + while (list1 && list2) { + file1 = list1->data; + file2 = list2->data; + + ret = strcmp(file1->filename, file2->filename); + + if (file2->is_deleted) { + list2 = g_list_next(list2); + continue; + } + + if (ret == 0) { + hash_assign(file2->hash, file1->hash); + list1 = g_list_next(list1); + list2 = g_list_next(list2); + } else if (ret < 0) { + list1 = g_list_next(list1); + } else { + list2 = g_list_next(list2); + } + } +} diff --git a/src/chroot.c b/src/chroot.c new file mode 100644 index 0000000..2878496 --- /dev/null +++ b/src/chroot.c @@ -0,0 +1,91 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +void chroot_create_full(int newversion) +{ + int ret; + char *group; + char * command; + char *full_dir; + + string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion); + + g_mkdir_with_parents(full_dir, S_IRWXU); + + + /* start with base */ + LOG(NULL, "Copying chroot os-core to full", ""); + string_or_die(&command, "rsync -aAX %s/%i/os-core/ %s", + image_dir, newversion, full_dir); + ret = system(command); + assert(ret==0); + free(command); + + /* overlay any new files from each group */ + while (1) { + group = next_group(); + if (!group) { + break; + } + + LOG(NULL, "Overlaying bundle chroot onto full", "%s", group); + string_or_die(&command, "rsync -aAX --ignore-existing %s/%i/%s/ %s", + image_dir, newversion, group, full_dir); + ret = system(command); + assert(ret==0); + free(command); + } + + /* and since some dumb files have things like install timestamps in + * them, we pull files from full back to bundles so each instance + * of the "same" file is truly identical across bundles. It may + * make sense at some point to add more logic here for conflict + * detection and resolution, including work to insure renames are + * consistent across groups. */ + while (1) { + group = next_group(); + if (!group) { + break; + } + + LOG(NULL, "Recopy bundle chroot out of full", "%s", group); + string_or_die(&command, "rsync -aAX --existing %s %s/%i/%s", + full_dir, image_dir, newversion, group); + ret = system(command); + assert(ret==0); + free(command); + } + free(full_dir); +} diff --git a/src/config.c b/src/config.c new file mode 100644 index 0000000..db9b014 --- /dev/null +++ b/src/config.c @@ -0,0 +1,111 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#include +#include +#include +#include +#include + +#include "swupd.h" + +static GKeyFile *keyfile = NULL; + + +char *config_image_base(void) +{ + assert(keyfile != NULL); + + return g_key_file_get_value(keyfile, "Server", "imagebase", NULL); +} + +char *config_output_dir(void) +{ + assert(keyfile != NULL); + + return g_key_file_get_value(keyfile, "Server", "outputdir", NULL); +} + +char *config_empty_dir(void) +{ + assert(keyfile != NULL); + + return g_key_file_get_value(keyfile, "Server", "emptydir", NULL); +} + +int config_initial_version(void) +{ + assert(keyfile != NULL); + char *c; + int version; + + c = g_key_file_get_value(keyfile, "Server", "initialversion", NULL); + + if (!c) { + return 0; + } + version = strtoull(c, NULL, 10); + free(c); + return version; +} + + +bool read_configuration_file(char *filename) +{ + GError *error = NULL; + + keyfile = g_key_file_new(); + + if (!g_key_file_load_from_file(keyfile, filename, G_KEY_FILE_NONE, &error)) { + printf("Failed to Load configuration file: %s (%s)\n", filename, error->message); + g_error_free(error); + return false; + } +#if 0 + char *c; + + printf("Configuration settings:\n"); + + c = config_image_base(); + printf(" image base path : %s\n", c); + free(c); + + c = config_output_dir(); + printf(" output directory : %s\n", c); + free(c); + + c = config_empty_dir(); + printf(" empty directory : %s\n", c); + free(c); + + printf("\n"); +#endif + return true; +} + +void release_configuration_data(void) +{ + if (keyfile != NULL) { + g_key_file_free(keyfile); + keyfile = NULL; + } +} diff --git a/src/delta.c b/src/delta.c new file mode 100644 index 0000000..8fb81bc --- /dev/null +++ b/src/delta.c @@ -0,0 +1,179 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "swupd.h" +#include "xattrs.h" + +void __create_delta(struct file *file, int from_version) +{ + char *original, *newfile, *outfile, *dotfile, *testnewfile, *sanitycheck; + char *conf; + int ret; + + if (file->is_link) { + return; + } + + if (file->is_deleted) { + return; /* file got deleted -> by definition we cannot make a delta */ + } + + if (file->is_dir) { + return; /* cannot do this for directories yet */ + } + + conf = config_image_base(); + string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename); + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + + free(conf); + + conf = config_output_dir(); + + string_or_die(&outfile, "%s/%i/delta/%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash); + string_or_die(&dotfile, "%s/%i/delta/.%i-%i-%s", conf, file->last_change, from_version, file->last_change, file->hash); + string_or_die(&testnewfile, "%s/%i/delta/.%i-%i-%s.testnewfile", conf, file->last_change, from_version, file->last_change, file->hash); + string_or_die(&sanitycheck, "cmp -s \"%s\" \"%s\"", newfile, testnewfile); + + LOG(file, "Making delta", "%s->%s", original, newfile); + + ret = xattrs_compare(original, newfile); + if (ret != 0) { + LOG(NULL, "xattrs have changed, don't create diff ", "%s", newfile); + goto out; + } + ret = make_bsdiff_delta(original, newfile, dotfile, 0); + if (ret < 0) { + LOG(file, "Delta creation failed", "%s->%s ret is %i", original, newfile, ret); + goto out; + } + if (ret == 1) { + LOG(file, "...delta larger than newfile: FULLDL", "%s", newfile); + unlink(dotfile); + goto out; + } + + /* does delta properly recreate expected content? */ + ret = apply_bsdiff_delta(original, testnewfile, dotfile); + if (ret != 0) { + printf("Delta application failed.\n"); + printf("Attempted %s->%s via diff %s\n", original, testnewfile, dotfile); + LOG(file, "Delta application failed.", "Attempted %s->%s via diff %s", original, testnewfile, dotfile); + +#warning the above is racy..tolerate it temporarily + // ok fine + //unlink(testnewfile); + //unlink(dotfile); + ret = 0; + goto out; + } + xattrs_copy(original, newfile); + + /* does xattrs have been correctly copied?*/ + if (xattrs_compare(original, testnewfile) != 0) { + printf("Delta application resulted in xattrs mismatch.\n"); + printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile); + LOG(file, "Delta xattrs mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile); + assert(0); + goto out; + } + + ret = system(sanitycheck); + if (ret == -1 || !WIFEXITED(ret) || WEXITSTATUS(ret) == 2) { + printf("Sanity check system command failed %i. \n", ret); + printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile); + assert(0); + goto out; + } else if (WEXITSTATUS(ret) == 1) { + printf("Delta application resulted in file mismatch %i. \n", ret); + printf("%s->%s via diff %s yielded %s\n", original, newfile, dotfile, testnewfile); + LOG(file, "Delta mismatch:", "%s->%s via diff %s yielded %s", original, newfile, dotfile, testnewfile); + +#warning this too will have failures due to races + //unlink(testnewfile); + //unlink(dotfile); + ret = 0; + goto out; + } + + unlink(testnewfile); + + if (rename(dotfile, outfile) != 0) { + if (errno == ENOENT) { + LOG(NULL, "dotfile:", " %s does not exist", dotfile); + } + LOG(NULL, "Failed to rename", ""); + } +out: free(sanitycheck); + free(testnewfile); + free(conf); + free(newfile); + free(original); + free(outfile); + free(dotfile); +} + +void prepare_delta_dir(struct manifest *manifest) +{ + char *path; + char *conf; + + printf("Preparing delta directory \n"); + + conf = config_output_dir(); + + + string_or_die(&path, "%s/%i", conf, manifest->version); + if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) { + if (errno != EEXIST) { + LOG(NULL, "Failed to create directory ", "%s", path); + return; + } + } + free(path); + string_or_die(&path, "%s/%i/delta/", conf, manifest->version); + if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) { + if (errno != EEXIST) { + LOG(NULL, "Failed to create directory ", "%s", path); + return; + } + } + + free(path); + free(conf); +} + diff --git a/src/fullfiles.c b/src/fullfiles.c new file mode 100644 index 0000000..284d551 --- /dev/null +++ b/src/fullfiles.c @@ -0,0 +1,348 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +/* output must be a file, which is a (compressed) tar file, of the file denoted by "file", without any of its + directory paths etc etc */ +static void create_fullfile(struct file *file) +{ + char *origin; + char *tarcommand=NULL; + char *tarname=NULL; + char *rename_source = NULL; + char *rename_target = NULL; + char *rename_tmpdir = NULL; + int ret; + struct stat sbuf; + char *empty, *indir, *outdir; + + + if (file->is_deleted) { + return; /* file got deleted -> by definition we cannot tar it up */ + } + + empty = config_empty_dir(); + indir = config_image_base(); + outdir = config_output_dir(); + + string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); + if (access(tarname, R_OK) == 0) { + /* output file already exists...done */ + free(tarname); + return; + } + free(tarname); + //printf("%s was missing\n", file->hash); + + string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); + if (lstat(origin, &sbuf) < 0) { + /* no input file: means earlier phase of update creation failed */ + LOG(NULL, "Failed to stat %s\n", origin); + assert(0); + } + + if (file->is_dir) { /* directories are easy */ + char *tmp1, *tmp2, *dir, *base; + + tmp1 = strdup(origin); + assert(tmp1); + base = basename(tmp1); + + tmp2 = strdup(origin); + assert(tmp2); + dir = dirname(tmp2); + + string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir); + if(!mkdtemp(rename_tmpdir)) { + LOG(NULL, "Failed to create temporary directory for %s move", origin); + assert(0); + } + + string_or_die(&tarcommand, "tar -C %s " TAR_PERM_ATTR_ARGS " -cf - '%s' --exclude='%s'/* 2> /dev/null | " + "tar -C %s " TAR_PERM_ATTR_ARGS " -xf - 2> /dev/null", + dir, base, base, rename_tmpdir); + if (system(tarcommand) != 0) { + LOG(NULL, "Failed to run command:", "%s", tarcommand); + assert(0); + } + free(tarcommand); + + string_or_die(&rename_source, "%s/%s", rename_tmpdir, base); + string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash); + if (rename(rename_source, rename_target)) { + LOG(NULL, "rename failed for %s to %s", rename_source, rename_target); + assert(0); + } + free(rename_source); + + /* for a directory file, tar up simply with gzip */ + string_or_die(&tarcommand, "tar -C %s %s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar", + rename_tmpdir, file->hash, outdir, file->last_change, file->hash); + if (system(tarcommand) != 0) { + LOG(NULL, "Failed to run command:", "%s", tarcommand); + assert(0); + } + free(tarcommand); + + if (rmdir(rename_target)) { + LOG(NULL, "rmdir failed for %s", rename_target); + } + free(rename_target); + if (rmdir(rename_tmpdir)) { + LOG(NULL, "rmdir failed for %s", rename_tmpdir); + } + + free(tmp1); + free(tmp2); + } else { /* files are more complex */ + char *gzfile = NULL, *bzfile = NULL, *xzfile = NULL; + char *tempfile; + uint64_t gz_size = LONG_MAX, bz_size = LONG_MAX, xz_size = LONG_MAX; + + /* step 1: hardlink the guy to an empty directory with the hash as the filename */ + string_or_die(&tempfile, "%s/%s", empty, file->hash); + if (link(origin, tempfile) < 0) { + LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile); + char *const argv[] = {"cp", "-a", origin, tempfile, NULL}; + if (system_argv(argv) != 0) { + assert(0); + } + } + + /* step 2a: tar it with each compression type */ + // lzma + string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -Jcf %s/%i/files/%s.tar.xz %s", + empty, outdir, file->last_change, file->hash, file->hash); + if (system(tarcommand) != 0) { + LOG(NULL, "Failed to run command:", "%s", tarcommand); + assert(0); + } + free(tarcommand); + // gzip + string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -zcf %s/%i/files/%s.tar.gz %s", + empty, outdir, file->last_change, file->hash, file->hash); + if (system(tarcommand) != 0) { + LOG(NULL, "Failed to run command:", "%s", tarcommand); + assert(0); + } + free(tarcommand); +#ifdef SWUPD_WITH_BZIP2 + string_or_die(&tarcommand, "tar --directory=%s " TAR_PERM_ATTR_ARGS " -jcf %s/%i/files/%s.tar.bz2 %s", + empty, outdir, file->last_change, file->hash, file->hash); + if (system(tarcommand) != 0) { + LOG(NULL, "Failed to run command:", "%s", tarcommand); + assert(0); + } + free(tarcommand); +#endif + + /* step 2b: pick the smallest of the three compression formats */ + string_or_die(&gzfile, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash); + if (stat(gzfile, &sbuf) == 0) { + gz_size = sbuf.st_size; + } + string_or_die(&bzfile, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash); + if (stat(bzfile, &sbuf) == 0) { + bz_size = sbuf.st_size; + } + string_or_die(&xzfile, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash); + if (stat(xzfile, &sbuf) == 0) { + xz_size = sbuf.st_size; + } + string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); + if (gz_size <= xz_size && gz_size <= bz_size) { + ret = rename(gzfile, tarname); + } else if (xz_size <= bz_size) { + ret = rename(xzfile, tarname); + } else { + ret = rename(bzfile, tarname); + } + if (ret != 0) { + LOG(file, "post-tar rename failed", "ret=%d", ret); + } + unlink(bzfile); + unlink(xzfile); + unlink(gzfile); + free(bzfile); + free(xzfile); + free(gzfile); + free(tarname); + + /* step 3: remove the hardlink */ + unlink(tempfile); + free(tempfile); + } + + free(indir); + free(outdir); + free(empty); + free(origin); +} + +static void create_fullfile_task(gpointer data, __unused__ gpointer user_data) +{ + struct file *file = data; + + create_fullfile(file); +} + +/* remove duplicate hashes from the fullfile creation list */ +static GList *get_deduplicated_fullfile_list(struct manifest *manifest) +{ + GList *list; + GList *outfiles = NULL; + struct file *file; + struct file *prev = NULL; + struct file *tmp; + + // presort by hash for easy deduplication + manifest->files = g_list_sort(manifest->files, file_sort_hash); + + list = g_list_first(manifest->files); + while (prev == NULL) { + tmp = list->data; + list = g_list_next(list); + + // find first new file + if (tmp->last_change == manifest->version) { + prev = tmp; + outfiles = g_list_prepend(outfiles, tmp); + } + } + while (list) { + file = list->data; + list = g_list_next(list); + + // add any new file having a unique hash + //FIXME: rename logic will be needed here + if (file->is_deleted || (file->last_change != manifest->version)) { + continue; + } + if (!hash_compare(prev->hash, file->hash)) { + outfiles = g_list_prepend(outfiles, file); + prev = file; + } + } + + // don't want sort by hash else hash[0] locks will be very conflicted + outfiles = g_list_sort(outfiles, file_sort_filename); + + return outfiles; +} + +static void submit_fullfile_tasks(GList *files) +{ + GThreadPool *threadpool; + GList *item; + struct file *file; + int ret; + int count=0; + GError *err = NULL; + + LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3); + threadpool = g_thread_pool_new(create_fullfile_task, NULL, + sysconf(_SC_NPROCESSORS_ONLN) * 3, + TRUE, NULL); + + printf("Starting downloadable fullfiles data creation\n"); + + item = g_list_first(files); + while (item) { + file = item->data; + item = g_list_next(item); + + ret = g_thread_pool_push(threadpool, file, &err); + if (ret == FALSE){ + printf("GThread create_fullfile_task push error\n"); + printf("%s\n", err->message); + assert(0); + } + count++; + } + printf("queued %i full file creations\n", count); + + printf("Waiting for downloadable fullfiles data creation to finish\n"); + g_thread_pool_free(threadpool, FALSE, TRUE); +} + +void create_fullfiles(struct manifest *manifest) +{ + GList *deduped_file_list; + char *path; + char *cmd; + char *conf, *empty; + + empty = config_empty_dir(); + + string_or_die(&cmd, "rm -rf %s", empty); + if (system(cmd) != 0) { + LOG(NULL, "Failed to run command:", "%s", cmd); + assert(0); + } + free(cmd); + g_mkdir_with_parents(empty, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + free(empty); + + conf = config_output_dir(); + + string_or_die(&path, "%s/%i", conf, manifest->version); + if(mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) { + if (errno != EEXIST) { + LOG(NULL, "Failed to create directory ", "%s", path); + return; + } + } + free(path); + string_or_die(&path, "%s/%i/files/", conf, manifest->version); + if (mkdir(path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) { + if (errno != EEXIST) { + LOG(NULL, "Failed to create directory ", "%s", path); + return; + } + } + free(path); + free(conf); + + /* De-duplicate the list of fullfiles needing created to avoid races */ + deduped_file_list = get_deduplicated_fullfile_list(manifest); + + /* Submit tasks to create full files */ + submit_fullfile_tasks(deduped_file_list); + + g_list_free(deduped_file_list); +} diff --git a/src/globals.c b/src/globals.c new file mode 100644 index 0000000..9f7ce6e --- /dev/null +++ b/src/globals.c @@ -0,0 +1,129 @@ +/* + * Software Updater - server side + * + * Copyright © 2015-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Timothy C. Pepper + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" + +int newversion = -1; +int minversion = 0; +char *format_string = NULL; +bool enable_signing = false; + +char *state_dir = NULL; +char *packstage_dir = NULL; +char *image_dir = NULL; +char *staging_dir = NULL; + +bool set_format_string(char *userinput) +{ + int version; + + // expect a positive integer + errno = 0; + version = strtoull(userinput, NULL, 10); + if ((errno < 0) || (version <= 0)) { + return false; + } + if (format_string) { + free(format_string); + } + string_or_die(&format_string, "%d", version); + + return true; +} + +bool set_state_dir(char *dir) +{ + if (dir == NULL || dir[0] == '\0') { + return false; + } + + /* TODO: more validation of input? */ + if (dir[0] != '/') { + printf("statedir must be a full path starting with '/', not '%c'\n", dir[0]); + return false; + } + + if (state_dir) { + free(state_dir); + } + string_or_die(&state_dir, "%s", dir); + + return true; +} + +bool init_globals(void) +{ + if (format_string == NULL) { + string_or_die(&format_string, "%d", SWUPD_DEFAULT_FORMAT); + } + + if (!init_state_globals()) { + return false; + } + + if (newversion == -1) { + printf("Missing version parameter: No new version number specified\n"); + return false; + } + + printf("file minversion == %d\n", minversion); + + return true; +} + +void free_globals(void) +{ + free(format_string); + free(state_dir); + free(packstage_dir); + free(image_dir); + free(staging_dir); +} + +bool init_state_globals(void) +{ + if (state_dir == NULL) { + string_or_die(&state_dir, "%s", SWUPD_SERVER_STATE_DIR); + } + string_or_die(&packstage_dir, "%s/%s", state_dir, "packstage"); + string_or_die(&image_dir, "%s/%s", state_dir, "image"); + string_or_die(&staging_dir, "%s/%s", state_dir, "www"); + + return true; +} + +void free_state_globals(void) +{ + free(state_dir); + free(packstage_dir); + free(image_dir); + free(staging_dir); +} diff --git a/src/groups.c b/src/groups.c new file mode 100644 index 0000000..fef3eab --- /dev/null +++ b/src/groups.c @@ -0,0 +1,88 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * Regis Merlino + * tkeel + * + */ + +#include +#include +#include +#include +#include + +#include "swupd.h" + +static GKeyFile *groupfile = NULL; + +static char **groups = NULL; +static gsize groupcount = 0; +static unsigned int groupcursor = 0; + +char *group_packages(char *group) +{ + assert(groupfile != NULL); + + return g_key_file_get_value(groupfile, group, "packages", NULL); +} + +char *group_groups(char *group) +{ + assert(groupfile != NULL); + + return g_key_file_get_value(groupfile, group, "groups", NULL); +} + + +void read_group_file(char *filename) +{ + GError *error = NULL; + groupfile = g_key_file_new(); + + if (g_key_file_load_from_file(groupfile, filename, G_KEY_FILE_NONE, &error)) { + groups = g_key_file_get_groups(groupfile, &groupcount); + printf("Found %li groups\n", groupcount); + } else { + printf("Failed to Load group file: %s (%s)\n", filename, error->message); + g_error_free(error); + } +} + +void release_group_file(void) +{ + g_strfreev(groups); + groups = NULL; + if (groupfile != NULL) { + g_key_file_free(groupfile); + groupfile = NULL; + } +} + + +char *next_group(void) +{ + if (groupcursor >= groupcount) { + groupcursor = 0; + return NULL; + } + + return groups[groupcursor++]; +} diff --git a/src/helpers.c b/src/helpers.c new file mode 100644 index 0000000..144523e --- /dev/null +++ b/src/helpers.c @@ -0,0 +1,199 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + + +FILE * fopen_exclusive(const char *filename) /* no mode, opens for write only */ +{ + int fd; + + fd = open(filename,O_CREAT | O_EXCL | O_RDWR , 00644); + if (fd < 0) { + printf("exclusive open failed, filename=\"%s\",err=\"%s\"\n", + filename, strerror(errno)); + LOG(NULL, "exclusive open failed", "\\*filename=\"%s\",err=\"%s\"*\\", + filename, strerror(errno)); + return NULL; + } + return fdopen(fd, "w"); +} + +void dump_file_info(struct file *file) +{ + printf("%s:\n", file->filename); + printf("\t%s\n", file->hash); + printf("\t%d\n", file->last_change); + + if (file->use_xattrs) { + printf("\tuse_xattrs\n"); + } + if (file->is_dir) { + printf("\tis_dir\n"); + } + if (file->is_file) { + printf("\tis_file\n"); + } + if (file->is_link) { + printf("\tis_link\n"); + } + if (file->is_deleted) { + printf("\tis_deleted\n"); + } + if (file->is_manifest) { + printf("\tis_manifest\n"); + } + if (file->is_config) { + printf("\tis_config\n"); + } + if (file->is_state) { + printf("\tis_state\n"); + } + if (file->is_boot) { + printf("\tis_boot\n"); + } + if (file->is_rename) { + printf("\tis_rename\n"); + } + + if (file->peer) { + printf("\tpeer %s(%s)\n", file->peer->filename, file->peer->hash); + } +} + +/* Tiny memory allocations don't fail or it's the end of the world. But + * source checkers like to see checked returns. But repeatedly checking + * returns introduces bugs. So here's the asprintf checking helper. */ +void string_or_die(char **strp, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + if (vasprintf(strp, fmt, ap) <= 0) { + assert(0); + } + va_end(ap); +} + +void print_elapsed_time(struct timeval *previous_time, struct timeval *current_time) +{ + char *elapsed; + + gettimeofday(current_time, NULL); + elapsed = get_elapsed_time(previous_time, current_time); + if (strcmp(elapsed, " ") == 0) { + printf("\t~0.0 elapsed for Manifest.full creation\n"); + } else { + printf("\t%s elapsed for full chroot creation\n", elapsed); + } + previous_time->tv_sec = current_time->tv_sec; + previous_time->tv_usec = current_time->tv_usec; + + free(elapsed); +} + +void concat_str_array(char **output, char *const argv[]) +{ + int size = 0; + + for (int i = 0; argv[i]; i++) { + size += strlen(argv[i]) + 1; + } + + *output = malloc(size + 1); + if (!*output) { + LOG(NULL, "Failed to allocate", "%i bytes", size); + assert(0); + } + strcpy(*output, ""); + for (int i = 0; argv[i]; i++) { + strcat(*output, argv[i]); + strcat(*output, " "); + } +} + +int system_argv(char *const argv[]) +{ + int child_exit_status; + pid_t pid; + int status; + + pid = fork(); + + if (pid == 0) { /* child */ + execvp(*argv, argv); + LOG(NULL, "This line must not be reached", ""); + assert(0); + } else if (pid < 0) { + LOG(NULL, "Failed to fork a child process", ""); + assert(0); + } else { + pid_t ws = waitpid(pid, &child_exit_status, 0); + + if (ws == -1) { + LOG(NULL, "Failed to wait for child process", ""); + assert(0); + } + + if (WIFEXITED(child_exit_status)) { + status = WEXITSTATUS(child_exit_status); + } else { + LOG(NULL, "Child process didn't exit", ""); + assert(0); + } + + if (status != 0) { + char* cmdline = NULL; + + concat_str_array(&cmdline, argv); + LOG(NULL, "Failed to run command:", "%s", cmdline); + free(cmdline); + } + + return status; + } +} + +void check_root(void) +{ + if (getuid() != 0) { + printf(PACKAGE_NAME " code really wants to be run as root at this point.\n"); + printf("However, currently it's not being run as root.. exiting\n"); + printf("\n"); + exit(EXIT_FAILURE); + } +} diff --git a/src/heuristics.c b/src/heuristics.c new file mode 100644 index 0000000..c3a4231 --- /dev/null +++ b/src/heuristics.c @@ -0,0 +1,127 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + + + +/* TODO: this needs to come from a file that has a list of regexps */ + +static void config_file_heuristics(struct file *file) +{ + if ((strncmp(file->filename, "/etc/", 5) == 0)) { + file->is_config = 1; + LOG(NULL, "Config file marked", "%s", file->filename); + } +} + +static void runtime_state_heuristics(struct file *file) +{ + /* these are shipped directories that are not themselves state, + * rather only their contents are state */ + if ((strcmp(file->filename, "/usr/src/debug") == 0) || + (strcmp(file->filename, "/dev") == 0) || + (strcmp(file->filename, "/home") == 0) || + (strcmp(file->filename, "/proc") == 0) || + (strcmp(file->filename, "/root") == 0) || + (strcmp(file->filename, "/run") == 0) || + (strcmp(file->filename, "/sys") == 0) || + (strcmp(file->filename, "/tmp") == 0) || + (strcmp(file->filename, "/var") == 0)) { + return; + } + + /* the contents of these directory are state, ideally this never + * triggers if our package builds are clean */ + if ((strncmp(file->filename, "/dev/", 5) == 0) || + (strncmp(file->filename, "/home/", 6) == 0) || + (strncmp(file->filename, "/proc/", 6) == 0) || + (strncmp(file->filename, "/root/", 6) == 0) || + (strncmp(file->filename, "/run/", 5) == 0) || + (strncmp(file->filename, "/sys/", 5) == 0) || + (strncmp(file->filename, "/tmp/", 5) == 0) || + (strncmp(file->filename, "/var/", 5) == 0) || + (strncmp(file->filename, "/usr/src/", 9) == 0)) { + file->is_state = 1; + LOG(NULL, "Bad package runtime state detected", "%s", file->filename); + return; + } + + /* these are commonly added directories for user customizatio, + * ideally this never triggers if our package builds are clean */ + if ((strncmp(file->filename, "/acct", 5) == 0) || + (strncmp(file->filename, "/cache", 6) == 0) || + (strncmp(file->filename, "/data", 5) == 0) || + (strncmp(file->filename, "/lost+found", 11) == 0) || + (strncmp(file->filename, "/mnt/asec", 9) == 0) || + (strncmp(file->filename, "/mnt/obb", 8) == 0) || + (strncmp(file->filename, "/mnt/shell/emulated", 19) == 0) || + (strncmp(file->filename, "/mnt/swupd", 10) == 0) || + (strncmp(file->filename, "/oem", 4) == 0) || + (strncmp(file->filename, "/system/rt/audio", 16) == 0) || + (strncmp(file->filename, "/system/rt/gfx", 14) == 0) || + (strncmp(file->filename, "/system/rt/media", 16) == 0) || + (strncmp(file->filename, "/system/rt/wifi", 15) == 0) || + (strncmp(file->filename, "/system/etc/firmware/virtual", 28) == 0)) { + file->is_state = 1; + LOG(NULL, "Surprising package runtime state detected", "%s", file->filename); + return; + } +} + +static void boot_file_heuristics(struct file *file) +{ + if ((strncmp(file->filename, "/boot/", 6) == 0) || + (strncmp(file->filename, "/usr/lib/modules/", 17) == 0) || + (strncmp(file->filename, "/usr/lib/kernel/", 16) == 0) || + (strncmp(file->filename, "/usr/share/kernel/", 18) == 0) || + (strncmp(file->filename, "/usr/lib/gummiboot", 18) == 0) || + (strncmp(file->filename, "/usr/bin/gummiboot", 18) == 0)) { + file->is_boot = 1; + // LOG(NULL, "Boot file marked", "%s", file->filename); + } +} + +void apply_heuristics(struct manifest *manifest) +{ + GList *list; + struct file *file; + + list = g_list_first(manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + + config_file_heuristics(file); + runtime_state_heuristics(file); + boot_file_heuristics(file); + } +} diff --git a/src/log.c b/src/log.c new file mode 100644 index 0000000..892930a --- /dev/null +++ b/src/log.c @@ -0,0 +1,165 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Tim Pepper + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +static FILE *logfile; + +static struct timeval start_time; + +void init_log(int version) +{ + char filename[4096]; + sprintf(filename, "swupd-server.log.%i", version); + logfile = fopen(filename, "w"); + gettimeofday(&start_time, NULL); +} +void init_log_stdout(void) +{ + logfile = stdout; + gettimeofday(&start_time, NULL); +} + +/* turn a time delta into a decimal string showing elapsed seconds, or an + * empty string if the time was less than 1ms */ +char *get_elapsed_time(struct timeval *t1, struct timeval *t2) +{ + char *elapsed = NULL; + double d = 0.0; + + // subtract off of "now" the swupd starting time + t2->tv_sec -= start_time.tv_sec; + while (t2->tv_usec < start_time.tv_usec) { + t2->tv_sec--; + t2->tv_usec += 1000000; + } + t2->tv_usec -= start_time.tv_usec; + + // if the start was not "0", subtract to get delta + if (t1->tv_usec || t1->tv_sec) { + d = t2->tv_sec * 1000000.0 + t2->tv_usec; + d -= t1->tv_sec * 1000000.0 + t1->tv_usec; + d = d / 1000000.0; + + // highlight costly operations by only printing > 1ms deltas + if (d > 0.001) { + string_or_die(&elapsed, "%5.3f", d); + } else { + string_or_die(&elapsed, " "); + } + } + return elapsed; +} + +void __log_message(struct file *file, char *msg, char *filename, int linenr, const char *fmt, ...) +{ + char *buf = NULL; + struct timeval current_time; + static struct timeval previous_time; + va_list ap; + char *logstring = NULL; + char filebuf[4096]; + char filebuf2[4096]; + + if (!logfile) { + return;\ + } + + gettimeofday(¤t_time, NULL); + logstring = get_elapsed_time(&previous_time, ¤t_time); + previous_time = current_time; + + va_start(ap, fmt); + if (vasprintf(&buf, fmt, ap) < 0) { + assert(0); + } + va_end(ap); + + filebuf[4095] = 0; + filebuf2[0] = 0; + strncpy(filebuf, filename, 4095); + if (file) { + strncpy(filebuf2, file->filename, 4095); + } + while (strlen(filebuf) < 29) { + strcat(filebuf, " "); + } + while (strlen(filebuf2) < 30) { + strcat(filebuf2, " "); + } + + fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", + (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf); + + free(logstring); + free(buf); + fflush(logfile); + +} + +void close_log(int version, int exit_status) +{ + struct timeval current_time; + int t_sec; + int t_msec; + + if (!logfile) { + return; + } + + gettimeofday(¤t_time, NULL); + + current_time.tv_sec -= start_time.tv_sec; + while (current_time.tv_usec < start_time.tv_usec) { + current_time.tv_sec--; + current_time.tv_usec += 1000000; + } + current_time.tv_usec -= start_time.tv_usec; + t_sec = (int)current_time.tv_sec; + t_msec = (int)current_time.tv_usec / 1000; + + LOG(NULL, "Update build duration", "%i.%03i seconds", t_sec, t_msec); + printf("\n\nUpdate build duration was %i.%03i seconds\n", t_sec, t_msec); + + if (exit_status == EXIT_SUCCESS) { + LOG(NULL, "Update build success", "version %i", version); + printf("Update complete. System update built for version %i\n", version); + } else { + LOG(NULL, "Update build failure", "version %i", version); + printf("Update build failed for version %i\n", version); + } + + fclose(logfile); + logfile = NULL; +} diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..2fccb3d --- /dev/null +++ b/src/main.c @@ -0,0 +1,465 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * Regis Merlino + * Tom Keel + * Tim Pepper + * + */ + + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" + +static void banner(void) +{ + printf(PACKAGE_NAME " update creator version " PACKAGE_VERSION "\n"); + printf(" Copyright (C) 2012-2016 Intel Corporation\n"); + printf("\n"); +} + +static const struct option prog_opts[] = { + {"help", no_argument, 0, 'h'}, + {"version", no_argument, 0, 'v'}, + {"osversion", required_argument, 0, 'o'}, + {"minversion", required_argument, 0, 'm'}, + {"format", required_argument, 0, 'F'}, + {"getformat", no_argument, 0, 'g'}, + {"statedir", required_argument, 0, 'S'}, + {"signcontent", no_argument, 0, 's'}, + {0, 0, 0, 0} +}; + +static void print_help(const char *name) { + printf("Usage:\n"); + printf(" %s [OPTION...]\n\n", name); + printf("Help Options:\n"); + printf(" -h, --help Show help options\n"); + printf(" -v, --version Show software version\n"); + printf("\n"); + printf("Application Options:\n"); + printf(" -o, --osversion The OS version for which to create an update\n"); + printf(" -m, --minversion Optional minimum file version to write into manifests per file\n"); + printf(" -F, --format Optional format string [ default:=%d ]\n", SWUPD_DEFAULT_FORMAT); + printf(" -g, --getformat Print current format string and exit\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); + printf("\n"); +} + +static bool parse_options(int argc, char **argv) +{ + int opt; + + while ((opt = getopt_long(argc, argv, "hvo:m:F:g:S:s", prog_opts, NULL)) != -1) { + switch (opt) { + case '?': + case 'h': + print_help(argv[0]); + return false; + case 'v': + banner(); + return false; + case 'o': + // expect positive integer + newversion = strtoull(optarg, NULL, 10); + if (newversion <= 0) { + printf("-o | --osversion == %d, requires value > 0 \n", newversion); + return false; + } + break; + case 'm': + minversion = strtoull(optarg, NULL, 10); + if (minversion < 0) { + printf("-m | --minversion == %d, require value >= 0 \n", newversion); + return false; + } + break; + case 'F': + if (!optarg || !set_format_string(optarg)) { + printf("Invalid --format argument\n\n"); + return false; + } + break; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument ''%s'\n\n", optarg); + return false; + } + break; + case 'g': + if (format_string == NULL) { + printf("%d\n", SWUPD_DEFAULT_FORMAT); + } else { + printf("%s\n", format_string); + free_globals(); + } + exit(0); + case 's': + enable_signing = true; + break; + } + } + + if (!init_globals()) { + return false; + } + + return true; +} + +static void populate_dirs(int version) +{ + char *newversiondir; + + string_or_die(&newversiondir, "%s/%d", image_dir, version); + + if ((access(newversiondir, F_OK | R_OK) != 0) && (version == 0)) { + char *latestpath = NULL; + + string_or_die(&latestpath, "%s/latest.version", image_dir); + + printf("** %s does not exist... creating and populating\n", newversiondir); + if (mkdir(newversiondir, 0755) != 0) { + printf("Failed to create directory\n"); + } + strncat(newversiondir, "/os-core", strlen("/os-core")); + if (mkdir(newversiondir, 0755) != 0) { + printf("Failed to create os-core subdirectory\n"); + } + + FILE *latestver; + latestver = fopen_exclusive(latestpath); + if (latestver == NULL) { + printf("Failed to create %s\n", latestpath); + free(latestpath); + return; + } + if (fwrite("0\n", 2, 1, latestver) != 1) { + LOG(NULL, "Failed to write latestver", "errno: %d", errno); + } + + free(latestpath); + fclose(latestver); + } + /* groups don't exist in version 0 */ + if (version != 0) { + char *group; + while (1) { + group = next_group(); + if (!group) { + break; + } + + string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group); + + /* Create the bundle directory(s) as needed */ + if (access(newversiondir, F_OK | R_OK) != 0) { + printf("%s does not exist...creating\n", group); + if (mkdir(newversiondir, 0755) != 0) { + printf("Failed to create %s subdirectory\n", group); + } + } + } + } + free(newversiondir); +} + +static int check_build_env(void) +{ + char *temp_dir = NULL; + string_or_die(&temp_dir, "%s/%s", state_dir, "/temp"); + + if (access(temp_dir, F_OK | R_OK) != 0) { + LOG(NULL, "%s", "does not exist...creating directory", temp_dir); + if (mkdir(temp_dir, 0755) != 0) { + printf("Failed to create build directory, EXITING\n"); + free(temp_dir); + return -errno; + } + } + free(temp_dir); + + return 0; +} + +int main(int argc, char **argv) +{ + struct manifest *new_core = NULL; + struct manifest *old_core = NULL; + + struct manifest *new_MoM = NULL; + struct manifest *old_MoM = NULL; + + struct manifest *old_full = NULL; + struct manifest *new_full = NULL; + + GList *manifests_last_versions_list = NULL; + int newfiles = 0; + + struct timeval current_time; + struct timeval previous_time; + + int exit_status = EXIT_FAILURE; + int ret; + + char *file_path = NULL; + + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + + if (!parse_options(argc, argv)) { + free_globals(); + return EXIT_FAILURE; + } + + banner(); + check_root(); + ret = check_build_env(); + if (ret != 0) { + printf("Failed to setup build environment: ERRNO = %i\n", ret); + goto exit; + } + + /* Initilize the crypto signature module */ + if (!signature_initialize()) { + printf("Can't initialize the crypto signature module!\n"); + goto exit; + } + + + string_or_die(&file_path, "%s/server.ini", state_dir); + if (!read_configuration_file(file_path)) { + printf("Failed to read %s configuration file!\n", state_dir); + free(file_path); + goto exit; + } + free(file_path); + + string_or_die(&file_path, "%s/groups.ini", state_dir); + read_group_file(file_path); + free(file_path); + + read_current_version("latest.version"); + printf("Last processed version is %i\n", current_version); + + populate_dirs(newversion); + + printf("Next version is %i \n", newversion); + init_log(newversion); + + gettimeofday(&previous_time, NULL); + + printf("Entering phase 1: full chroot and manifest preparation\n"); + + /* Step 1: Create "full" chroot (a union of _everything_ in the build) + * Presume an external entity created chroots for all bundles. */ + printf("Syncing bundle chroots with full chroot\n"); + chroot_create_full(newversion); + + print_elapsed_time(&previous_time, ¤t_time); + + printf("Calculating full manifest (this is expensive/slow)\n"); + old_full = manifest_from_file(current_version, "full"); + new_full = full_manifest_from_directory(newversion); + apply_heuristics(old_full); + apply_heuristics(new_full); + match_manifests(old_full, new_full); + apply_heuristics(new_full); +#warning disabled rename detection for some simplicity +/* rename_detection(new_full); */ + + print_elapsed_time(&previous_time, ¤t_time); + + printf("Entering phase 2: Doing the os-core bundle\n"); + /* Phase 2 : the os-core set */ + + /* Step 2: Make a manifest for the os-core set */ + old_MoM = manifest_from_file(current_version, "MoM"); + new_MoM = alloc_manifest(newversion, "MoM"); + old_core = manifest_from_file(manifest_subversion(old_MoM, "os-core"), "os-core"); + new_core = sub_manifest_from_directory("os-core", newversion); + add_component_hashes_to_manifest(new_core, new_full); + + new_core->prevversion = old_core->version; + apply_heuristics(old_core); + apply_heuristics(new_core); + new_MoM->prevversion = old_MoM->version; + + manifests_last_versions_list = get_last_versions_list(newversion, SWUPD_NUM_MANIFEST_DELTAS); + + /* Step 3: Compare to the previous core manifest... */ + + if (match_manifests(old_core, new_core) == 0) { + LOG(NULL, "Core component has not changed, no new manifest", ""); + printf("Core component has not changed, no new manifest\n"); + /* Step 3b: ... and if nothing is new, we stay at the old version */ + new_core->version = old_core->version; + } else { + apply_heuristics(new_core); + /* Step 3c: ... else save the manifest */ + type_change_detection(new_core); + + /* Detect renamed files specifically for each pack */ +#warning disabled rename detection for some simplicity +/* rename_detection(...); */ + + sort_manifest_by_version(new_core); + newfiles = prune_manifest(new_core); + if (newfiles <= 0) { + LOG(NULL, "", "Core component has not changed (after pruning), exiting"); + printf("Core component has not changed (after pruning), exiting\n"); + goto exit; + } + LOG(NULL, "", "Core component has changes (%d), writing out new manifest", newfiles); + printf("Core component has changes (%d), writing out new manifest\n", newfiles); + if (write_manifest(new_core) != 0) { + LOG(NULL, "", "Core component manifest write failed"); + printf("Core component manifest write failed\n"); + goto exit; + } + create_manifest_deltas(new_core, manifests_last_versions_list); + } + + nest_manifest(new_MoM, new_core); + + /* Phase 3: the functional bundles */ + printf("Entering phase 3: The bundles\n"); + while (1) { + char *group = next_group(); + struct manifest *oldm; + struct manifest *newm; + + if (!group) { + break; + } + + if (strcmp(group, "os-core") == 0) { + continue; + } + + printf("Processing bundle %s\n", group); + + /* Step 4: Make a manifest for this functonal group */ + oldm = manifest_from_file(manifest_subversion(old_MoM, group), group); + newm = sub_manifest_from_directory(group, newversion); + add_component_hashes_to_manifest(newm, new_full); + apply_heuristics(oldm); + apply_heuristics(newm); + newm->prevversion = oldm->version; + + /* Step 5: Subtract the core files from the manifest */ + subtract_manifests(oldm, old_core); + subtract_manifests(newm, new_core); + + /* Step 6: Compare manifest to the previous version... */ + if (match_manifests(oldm, newm) == 0) { + LOG(NULL, "", "%s components have not changed, no new manifest", group); + printf("%s components have not changed, no new manifest\n", group); + /* Step 6a: if nothing changed, stay at the old version */ + newm->version = oldm->version; + } else { + apply_heuristics(newm); +#warning missing rename_detection here + /* Step 6b: otherwise, write out the manifest */ + sort_manifest_by_version(newm); + type_change_detection(newm); + newfiles = prune_manifest(newm); + if (newfiles > 0) { + LOG(NULL, "", "%s component has changes (%d), writing out new manifest", group, newfiles); + printf("%s component has changes (%d), writing out new manifest\n", group, newfiles); + if (write_manifest(newm) != 0) { + LOG(NULL, "", "%s component manifest write failed", group); + printf("%s component manifest write failed\n", group); + goto exit; + } + create_manifest_deltas(newm, manifests_last_versions_list); + } else { + LOG(NULL, "", "%s component has not changed (after pruning), no new manifest", group); + printf("%s component has not changed (after pruning), no new manifest\n", group); + newm->version = oldm->version; + } + } + + nest_manifest(new_MoM, newm); + } + + print_elapsed_time(&previous_time, ¤t_time); + + printf("Entering phase 4: completion manifests \n"); + /* Phase 4 : manifest completion */ + /* Step 7: Create the meta-manifest */ + sort_manifest_by_version(new_MoM); + if (write_manifest(new_MoM) != 0) { + LOG(NULL, "Failed to write new MoM", ""); + printf("Failed to write new MoM\n"); + goto exit; + } + create_manifest_deltas(new_MoM, manifests_last_versions_list); + + print_elapsed_time(&previous_time, ¤t_time); + + printf("Entering phase 5: creating download content\n"); + /* Phase 5: wrapping up */ + + +//TODO: should be fixup_versions() and preceed all write_manifest() calls + maximize_to_full(new_MoM, new_full); + + sort_manifest_by_version(new_full); + prune_manifest(new_full); + if (write_manifest(new_full) != 0) { + goto exit; + } + + /* Step 8: Prepare delta directory */ + prepare_delta_dir(new_full); + + print_elapsed_time(&previous_time, ¤t_time); + + exit_status = EXIT_SUCCESS; +exit: + if (exit_status == EXIT_SUCCESS) { + write_cookiecrumbs_to_download_area(newversion); + //write_new_version("latest.version", newversion); + } + release_configuration_data(); + release_group_file(); + signature_terminate(); + g_list_free(manifests_last_versions_list); + + close_log(newversion, exit_status); + printf("Update creation %s\n", exit_status == EXIT_SUCCESS ? "complete" : "failed"); + + free_globals(); + return exit_status; +} diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c new file mode 100644 index 0000000..6244bf1 --- /dev/null +++ b/src/make_fullfiles.c @@ -0,0 +1,121 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +static const struct option prog_opts[] = { + {"help", no_argument, 0, 'h'}, + {"statedir", required_argument, 0, 'S'}, + {0, 0, 0, 0} +}; + +static void usage(const char *name) +{ + printf("usage:\n"); + printf(" %s \n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf("\n"); +} + +static bool parse_options(int argc, char **argv) +{ + int opt; + + while ((opt = getopt_long(argc, argv, "hS:", prog_opts, NULL)) != -1) { + switch (opt) { + case '?': + case 'h': + usage(argv[0]); + return false; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument '%s'\n\n", optarg); + return false; + } + break; + } + } + + if (!init_state_globals()) { + return false; + } + + return true; +} + +static void banner(void) +{ + printf(PACKAGE_NAME " update creator -- fullfiles -- version " PACKAGE_VERSION "\n"); + printf(" Copyright (C) 2012-2016 Intel Corporation\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + struct manifest *manifest; + int version; + char *file_path = NULL; + + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; + } + banner(); + check_root(); + + string_or_die(&file_path, "%s/server.ini", state_dir); + read_configuration_file(file_path); + free(file_path); + + if (argc - optind != 1) { + usage(argv[0]); + free_state_globals(); + exit(EXIT_FAILURE); + } + + version = strtoull(argv[optind++], NULL, 10); + if (version < 0) { + printf("Usage:\n\tswupd_make_fullfiles \n\n"); + exit(EXIT_FAILURE); + } + + manifest = manifest_from_file(version, "full"); + create_fullfiles(manifest); + + free_state_globals(); + + return EXIT_SUCCESS; +} diff --git a/src/make_packs.c b/src/make_packs.c new file mode 100644 index 0000000..ba8ecff --- /dev/null +++ b/src/make_packs.c @@ -0,0 +1,166 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "swupd.h" + +static void banner(void) +{ + printf(PACKAGE_NAME "update pack creator version " PACKAGE_VERSION "\n"); + printf(" Copyright (C) 2012-2016 Intel Corporation\n"); + printf("\n"); +} + +static const struct option prog_opts[] = { + {"help", no_argument, 0, 'h'}, + {"statedir", required_argument, 0, 'S'}, + {"signcontent", no_argument, 0, 's'}, + {0, 0, 0, 0} +}; + +static void usage(const char *name) +{ + printf("usage:\n"); + printf(" %s \n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); + printf("\n"); +} + +static bool parse_options(int argc, char **argv) +{ + int opt; + + while ((opt = getopt_long(argc, argv, "hS:", prog_opts, NULL)) != -1) { + switch (opt) { + case '?': + case 'h': + usage(argv[0]); + return false; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument ''%s'\n\n", optarg); + return false; + } + break; + case 's': + enable_signing = true; + break; + } + } + + /* FIXME: *_state_globals() are ugly hacks */ + if (!init_state_globals()) { + return false; + } + + return true; +} + +int main(int argc, char **argv) +{ + int start_version; + long end_version; + //long version; + char *module; + struct packdata *pack; + int exit_status = EXIT_FAILURE; + char *file_path = NULL; + + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; + } + banner(); + check_root(); + + if (argc - optind != 3) { + usage(argv[0]); + exit(EXIT_FAILURE); + } + + /* FIXME: should use "end_version" not "0" and a unique filename + init_log(0); + */ + + /* Initilize the crypto signature module */ + if (!signature_initialize()) { + printf("Can't initialize the crypto signature module!\n"); + return exit_status; + } + + string_or_die(&file_path, "%s/server.ini", state_dir); + read_configuration_file(file_path); + free(file_path); + + start_version = strtoull(argv[optind++], NULL, 10); + end_version = strtoull(argv[optind++], NULL, 10); + module = argv[optind++]; + + if ((start_version < 0) || + (end_version == 0) || + (start_version >= end_version)) { + printf("Invalid version combination: %i - %li \n", start_version, end_version); + exit(EXIT_FAILURE); + } + + printf("Making pack-%s %i to %li\n", module, start_version, end_version); + + pack = calloc(1, sizeof(struct packdata)); + if (pack == NULL) { + assert(0); + } + + pack->module = module; + pack->from = start_version; + pack->to = end_version; + + if (make_pack(pack) == 0) { + exit_status = EXIT_SUCCESS; + } + + signature_terminate(); + + printf("Pack creation %s (pack-%s %i to %li)\n", + exit_status == EXIT_SUCCESS ? "complete" : "failed", + module, start_version, end_version); + + free_state_globals(); + return exit_status; +} diff --git a/src/manifest.c b/src/manifest.c new file mode 100644 index 0000000..68b9126 --- /dev/null +++ b/src/manifest.c @@ -0,0 +1,1203 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * tkeel + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "swupd.h" +#include "xattrs.h" + +#define LINK_SIZE_HINT 1024 +#define DIR_SIZE_HINT 1024 + +int file_sort_hash(gconstpointer a, gconstpointer b) +{ + struct file *A, *B; + A = (struct file *) a; + B = (struct file *) b; + + /* this MUST be a memcmp(), not bcmp() */ + return memcmp(A->hash, B->hash, SWUPD_HASH_LEN-1); +} + +int file_sort_version(gconstpointer a, gconstpointer b) +{ + struct file *A, *B; + A = (struct file *) a; + B = (struct file *) b; + + if (A->last_change < B->last_change) { + return -1; + } + if (A->last_change > B->last_change) { + return 1; + } + + return strcmp(B->filename, A->filename); +} + +int file_sort_filename(gconstpointer a, gconstpointer b) +{ + struct file *A, *B; + int ret; + A = (struct file *) a; + B = (struct file *) b; + + ret = strcmp(A->filename, B->filename); + if (ret) { + return ret; + } + if (A->is_deleted > B->is_deleted) { + return 1; + } + if (A->is_deleted < B->is_deleted) { + return -1; + } + + return 0; +} + +struct manifest *alloc_manifest(int version, char *component) +{ + struct manifest *manifest; + + manifest = calloc(1, sizeof(struct manifest)); + if (manifest == NULL) { + assert(0); + } + + manifest->version = version; + manifest->component = strdup(component); + + return manifest; +} + + +struct manifest *manifest_from_file(int version, char *component) +{ + FILE *infile; + char line[8192], *c, *c2; + int count = 0; + struct manifest *manifest; + char *filename, *conf; + int previous = 0; + + conf = config_output_dir(); + if (conf == NULL) { + assert(0); + } + + string_or_die(&filename, "%s/%i/Manifest.%s", conf, version, component); + free(conf); + + LOG(NULL, "Reading manifest", "%s", filename); + infile = fopen(filename, "rb"); + + if (infile == NULL) { + LOG(NULL, "Cannot read manifest", "%s (%s)\n", filename, strerror(errno)); + free(filename); + return alloc_manifest(version, component); + } + + /* line 1: MANIFEST\t */ + line[0] = 0; + if (fgets(line, 8191, infile) == NULL) { + fclose(infile); + return NULL; + } + + if (strncmp(line, "MANIFEST\t", 9)!=0) { + printf("Invalid file format: MANIFEST line\n"); + fclose(infile); + return NULL; + } + c = &line[9]; + if (strtoull(c, NULL, 10) == 0) { + printf("Unknown file format version in MANIFEST line: %s\n", c); + fclose(infile); + return NULL; + } + line[0] = 0; + while (strcmp(line, "\n")!=0) { + /* read the header */ + line[0] = 0; + if (fgets(line, 8191, infile) == NULL) { + break; + } + c = strchr(line, '\n'); + if (c) { + *c = 0; + } + if (strlen(line) == 0) { + break; + } + c = strchr(line, '\t'); + /* Make sure we're not at the end of the array before incrementing */ + if (c && c <= &line[strlen(line)]) { + c++; + } else { + printf("Manifest is corrupt\n"); + assert(0); + } + + if (strncmp(line,"version:", 8) == 0) { + version = strtoull(c, NULL, 10); + } + if (strncmp(line,"previous:", 9) == 0) { + previous = strtoull(c, NULL, 10); + } + } + + manifest = alloc_manifest(version, component); + manifest->prevversion = previous; + + /* empty line */ + while (!feof(infile)) { + struct file *file; + + line[0] = 0; + if (fgets(line, 8191, infile) == NULL) { + break; + } + c = strchr(line, '\n'); + if (c) { + *c = 0; + } + if (strlen(line) == 0) { + break; + } + + file = calloc(1, sizeof(struct file)); + if (file == NULL) { + assert(0); + } + c = line; + + c2 = strchr(c, '\t'); + if (c2) { + *c2 = 0; + c2++; + } + + if (c[0] == 'F') { + file->is_file = 1; + } else if (c[0] == 'D') { + file->is_dir = 1; + } else if (c[0] == 'L') { + file->is_link = 1; + } else if (c[0] == 'M') { + LOG(NULL, "Found a manifest!", "%s", c); + file->is_manifest = 1; + } else if (c[0] != '.') { + assert(0); /* unknown file type */ + } + + if (c[1] == 'd') { + file->is_deleted = 1; + } else if (c[1] != '.') { + assert(0); /* unknown deleted status */ + } + + if (c[2] == 'C') { + file->is_config = 1; + } else if (c[2] == 's') { + file->is_state = 1; + } else if (c[2] == 'b') { + file->is_boot = 1; + } else if (c[2] != '.') { + assert(0); /* unknown modifier status */ + } + + if (c[3] == 'r') { + file->is_rename = 1; + } else if (c[3] != '.') { + ; /* field 4: ignore unknown letters */ + } + + c = c2; + if (!c) { + free(file); + continue; + } + c2 = strchr(c, '\t'); + if (c2) { + *c2 = 0; + c2++; + } + + hash_assign(c, file->hash); + + c = c2; + if (!c) { + free(file); + continue; + } + c2 = strchr(c, '\t'); + if (c2) { + *c2 = 0; + c2++; + } + + file->last_change = strtoull(c, NULL, 10); + + c = c2; + if (!c) { + free(file); + continue; + } + file->filename = strdup(c); + + if (file->is_manifest) { + nest_manifest_file(manifest, file); + } else { + manifest->files = g_list_prepend(manifest->files, file); + } + manifest->count++; + count ++; + } + + manifest->files = g_list_sort(manifest->files, file_sort_filename); + fclose(infile); + LOG(NULL, "Manifest info", "Manifest for version %i/%s contains %i files", version, component, count); + free(filename); + return manifest; +} + + +void free_manifest(struct manifest *manifest) +{ + struct file *file; + + if (!manifest) { + return; + } + + while (manifest->files) { + file = manifest->files->data; + free(file->filename); + free(file); + manifest->files = g_list_delete_link(manifest->files, manifest->files); + } + free(manifest); +} + + +/* +backfill the "last changed" of each file in a manifest +by comparing the hash against the same file in the previous manifest + +in theory this is a O(N^2) operation, but due to sorting and new files being rare, +this is more like O(2N) in practice. +*/ + +int match_manifests(struct manifest *m1, struct manifest *m2) +{ + GList *list1, *list2; + struct file *file1, *file2; + int must_sort = 0; + int count = 0; + int first = 1; + + if (!m1) { + printf("Matching manifests up failed: No old manifest!\n"); + return -1; + } + + if (!m2) { + printf("Matching manifests up failed: No new manifest!\n"); + return -1; + } + + m1->files = g_list_sort(m1->files, file_sort_filename); + m2->files = g_list_sort(m2->files, file_sort_filename); + + list1 = g_list_first(m1->files); + list2 = g_list_first(m2->files); + + while (list1 && list2) { + int ret; + file1 = list1->data; + file2 = list2->data; + + file1->peer = NULL; + file2->peer = NULL; + + + ret = strcmp(file1->filename, file2->filename); + if (ret == 0) { + if (file1->is_deleted && file2->is_deleted && file1->is_rename) { + file2->is_rename = file1->is_rename; + hash_assign(file1->hash, file2->hash); + } + + if (hash_compare(file1->hash, file2->hash) && + file1->is_dir == file2->is_dir && + file1->is_link == file2->is_link && + file1->is_deleted == file2->is_deleted && + file1->is_file == file2->is_file && + file1->is_config == file2->is_config && + file1->is_state == file2->is_state && + file1->is_boot == file2->is_boot && + file1->last_change >= minversion) { + file2->last_change = file1->last_change; + file2->is_rename = file1->is_rename; + } else { + account_changed_file(); + if (first) { + LOG(file1, "file changed", ""); + first = 0; + } + count++; + } + + if (!file1->is_deleted || file2->is_deleted) { + file1->peer = file2; + file2->peer = file1; + } + + list1 = g_list_next(list1); + list2 = g_list_next(list2); + continue; + } + if (first) { + LOG(file1, "file added? ", "(file2 is %s)", file2->filename); + first = 0; + } + if (ret < 0) { + struct file *file3; +/* + * if we get here, file1 got deleted... what we must do is add a file entry for it in the target list. + * However, since we're currently walking the list we HAVE to prepend the entry.. and mark for sort at + * the end. + */ + file3 = calloc(1, sizeof(struct file)); + if (file3 == NULL) { + assert(0); + } + + file3->filename = strdup(file1->filename); + hash_set_zeros(file3->hash); + file3->is_deleted = 1; + file3->is_config = file1->is_config; + file3->is_state = file1->is_state; + file3->is_boot = file1->is_boot; + + if (!file1->is_deleted) { + file3->last_change = m2->version; + } else { + file3->last_change = file1->last_change; + file3->is_rename = file1->is_rename; + hash_assign(file1->hash, file3->hash); + } + + file3->peer = file1; + file1->peer = file3; + + list1 = g_list_next(list1); + m2->files = g_list_prepend(m2->files, file3); + m2->count++; + if (!file1->is_deleted) { + account_deleted_file(); + count++; + if (first) { + LOG(file1, "file got deleted", ""); + first = 0; + } + } + must_sort = 1; + continue; + } + /* if we get here, ret is > 0, which means this is a new file added */ + /* all we do is advance the pointer */ + account_new_file(); + list2 = g_list_next(list2); + count++; + } + +/* now deal with the tail ends */ + while (list1) { + file1 = list1->data; + + struct file *file3; + + if (first) { + LOG(file1, "file changed tail", ""); + first = 0; + } + count++; +/* + * if we get here, file1 got deleted... what we must do is add a file entry for it in the target list. + * However, since we're currently walking the list we HAVE to prepend the entry.. and mark for sort at + * the end. + */ + file3 = calloc(1, sizeof(struct file)); + if (file3 == NULL) { + assert(0); + } + + file3->filename = strdup(file1->filename); + hash_set_zeros(file3->hash); + file3->is_deleted = 1; + file3->is_config = file1->is_config; + file3->is_state = file1->is_state; + file3->is_boot = file1->is_boot; + + if (!file1->is_deleted) { + file3->last_change = m2->version; + } else { + file3->last_change = file1->last_change; + file3->is_rename = file1->is_rename; + hash_assign(file1->hash, file3->hash); + } + + file3->peer = file1; + file1->peer = file3; + + list1 = g_list_next(list1); + m2->files = g_list_prepend(m2->files, file3); + m2->count++; + if (!file1->is_deleted) { + account_deleted_file(); + } + must_sort = 1; + } + + while (list2) { + account_new_file(); + list2 = g_list_next(list2); + if (first) { + first = 0; + } + count++; + } + if (must_sort) { + m2->files = g_list_sort(m2->files, file_sort_filename); + } + + return count; +} + + +/* removes all files from m2 from m1 */ +void subtract_manifests(struct manifest *m1, struct manifest *m2) +{ + GList *list1, *list2; + struct file *file1, *file2; + + if (!m1) { + printf("Subtracting manifests failed: No m1 manifest!\n"); + return; + } + + if (!m2) { + printf("Subtracting manifests failed: No m2 manifest!\n"); + return; + } + + m1->files = g_list_sort(m1->files, file_sort_filename); + m2->files = g_list_sort(m2->files, file_sort_filename); + + list1 = g_list_first(m1->files); + list2 = g_list_first(m2->files); + + while (list1 && list2) { + int ret; + file1 = list1->data; + file2 = list2->data; + + ret = strcmp(file1->filename, file2->filename); + if (ret == 0) { + GList *todel; + todel = list1; + list1 = g_list_next(list1); + list2 = g_list_next(list2); + + if (file1->is_deleted == file2->is_deleted && file1->is_file == file2->is_file) { + m1->files = g_list_delete_link(m1->files, todel); + m1->count --; + continue; + } + } + + if (ret < 0) { + list1 = g_list_next(list1); + continue; + } + list2 = g_list_next(list2); + } +} + + + + +char *file_type_to_string(struct file *file) +{ + static char type[5]; + + strcpy(type, "...."); + + if (file->is_dir) { + type[0] = 'D'; + } + + if (file->is_link) { + type[0] = 'L'; + } + if (file->is_file) { + type[0] = 'F'; + } + if (file->is_manifest) { + type[0] = 'M'; + } + + if (file->is_deleted) { + type[1] = 'd'; + } + + if (file->is_config) { + type[2] = 'C'; + } + if (file->is_state) { + type[2] = 's'; + } + if (file->is_boot) { + type[2] = 'b'; + } + + if (file->is_rename) { + type[3] = 'r'; + } + + return type; +} + +static void compute_content_size(struct manifest *manifest) +{ + /* FIXME: this is a temporary implementation based on worst case */ + + GList *list; + struct file *file; + struct manifest *submanifest; + + list = g_list_first(manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + if (!file->is_deleted && (file->last_change == manifest->version)) { + if (file->is_file) { + manifest->contentsize += file->stat.st_size; + } else if (file->is_link) { + manifest->contentsize += LINK_SIZE_HINT; + } else if (file->is_dir) { + manifest->contentsize += DIR_SIZE_HINT; + } + } + } + + list = g_list_first(manifest->submanifests); + while (list) { + submanifest = list->data; + list = g_list_next(list); + + /* Do not take into account groups not included in download content */ + if (create_download_content_for_group(submanifest->component)) { + manifest->contentsize += submanifest->contentsize; + } + } +} + +/* Returns 0 == success, -1 == failure */ +static int write_manifest_signature(struct manifest *manifest, const char *suffix) +{ + char *conf = config_output_dir(); + char *filename = NULL; + int ret = -1; + + if (conf == NULL) { + assert(0); + } + string_or_die(&filename, "%s/%i/Manifest.%s%s", conf, manifest->version, + manifest->component, suffix); + if (!signature_sign(filename)) { + fprintf(stderr, "Creating signature for '%s' failed\n", filename); + goto exit; + } + ret = 0; +exit: + free(filename); + free(conf); + return ret; +} + +/* Returns 0 == success, -1 == failure */ +static int write_manifest_plain(struct manifest *manifest) +{ + GList *list; + struct file *file; + FILE *out = NULL; + char *base = NULL, *dir; + char *conf = config_output_dir(); + char *filename = NULL; + char *submanifest_filename = NULL; + int ret = -1; + + if (conf == NULL) { + assert(0); + } + string_or_die(&filename, "%s/%i/Manifest.%s", conf, manifest->version, manifest->component); + + base = strdup(filename); + if (base == NULL) { + assert(0); + } + dir = dirname(base); + + if (g_mkdir_with_parents(dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) != 0) { + assert(0); + } + + out = fopen(filename, "w"); + if (out == NULL) { + printf("Failed to open %s for write\n", filename); + goto exit; + } + + fprintf(out, "MANIFEST\t%s\n", format_string); + fprintf(out, "version:\t%i\n", manifest->version); + fprintf(out, "previous:\t%i\n", manifest->prevversion); + fprintf(out, "filecount:\t%i\n", manifest->count); + fprintf(out, "timestamp:\t%i\n", (int)time(NULL)); + compute_content_size(manifest); + fprintf(out, "contentsize:\t%llu\n", (long long unsigned int)manifest->contentsize); + fprintf(out, "\n"); + + list = g_list_first(manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + + fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename); + } + + list = g_list_first(manifest->manifests); + while (list) { + file = list->data; + list = g_list_next(list); + + string_or_die(&submanifest_filename, "%s/%i/Manifest.%s", conf, file->last_change, file->filename); + populate_file_struct(file, submanifest_filename); + ret = compute_hash(file, submanifest_filename); + if (ret != 0) { + printf("Hash computation failed\n"); + assert(0); + } + + fprintf(out, "%s\t%s\t%i\t%s\n", file_type_to_string(file), file->hash, file->last_change, file->filename); + free(submanifest_filename); + } + + ret = 0; +exit: + if (out) { + fclose(out); + } + free(conf); + free(base); + free(filename); + return ret; +} + +/* Returns 0 == success, -1 == failure */ +static int write_manifest_tar(struct manifest *manifest) +{ + char *conf = config_output_dir(); + char *tarcmd = NULL; + int ret = -1; + + if (conf == NULL) { + assert(0); + } + + /* now, tar the thing up for efficient full file download */ + /* and put the signature of the plain manifest into the archive, too */ + if (enable_signing) { + string_or_die(&tarcmd, "tar --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf " + "%s/%i/Manifest.%s.tar Manifest.%s Manifest.%s.signed", + conf, manifest->version, conf, manifest->version, manifest->component, + manifest->component, manifest->component); + } else { + string_or_die(&tarcmd, "tar --directory=%s/%i " TAR_PERM_ATTR_ARGS " -Jcf " + "%s/%i/Manifest.%s.tar Manifest.%s", + conf, manifest->version, conf, manifest->version, manifest->component, + manifest->component); + } + + if (system(tarcmd) != 0) { + fprintf(stderr, "Creation of Manifest.tar failed\n"); + goto exit; + } + ret = 0; +exit: + free(tarcmd); + free(conf); + return ret; +} + +bool create_download_content_for_group(const char *group) +{ +#warning should find a generic way to exclude some groups to pack creation + if (strcmp(group, "esp")) { + return true; + } + + return false; +} + +bool compute_hash_with_xattrs(const char *filename) +{ +#warning should find a generic way to specify if we should not use xattrs for calculating the hashes of some groups + if (strncmp(filename, "/boot/", 6) == 0) { + return true; + } + + return false; +} + +/* Returns 0 == success, -1 == failure */ +int write_manifest(struct manifest *manifest) +{ + if (write_manifest_plain(manifest) == 0 && + write_manifest_signature(manifest, "") == 0 && + write_manifest_tar(manifest) == 0 && + write_manifest_signature(manifest, ".tar") == 0) { + return 0; + } + return -1; +} + +void sort_manifest_by_version(struct manifest *manifest) +{ + if (manifest->files) { + manifest->files = g_list_sort(manifest->files, file_sort_version); + } + if (manifest->manifests) { + manifest->manifests = g_list_sort(manifest->manifests, file_sort_version); + } +} + +/* Conditionally remove some things from a manifest. + * Returns > 0 when the pruned manifest has new files. + * Returns 0 when the pruned manifest no longer has new files. + * Returns -1 on error. + */ +int prune_manifest(struct manifest *manifest) +{ + GList *list; + GList *next; + struct file *file; + int newfiles = 0; + + /* prune some files */ + list = g_list_first(manifest->files); + while (list) { + next = g_list_next(list); + file = list->data; + + if (OS_IS_STATELESS && (!file->is_deleted) && (file->is_config)) { + // toward being a stateless OS + LOG(file, "Skipping config file in manifest write", "component %s", manifest->component); + manifest->files = g_list_delete_link(manifest->files, list); + manifest->count--; + } else if (file->is_boot && file->is_deleted) { + // only expose the current best boot files, a client side entity can manage /boot's actual contents + // LOG(file, "Skipping deleted boot file in manifest write", "component %s", manifest->component); + manifest->files = g_list_delete_link(manifest->files, list); + manifest->count--; + } + list = next; + } + + /* check we haven't pruned all the new files */ + list = g_list_first(manifest->files); + while (list) { + next = g_list_next(list); + file = list->data; + list = next; + + if (file->last_change == manifest->version) { + newfiles++; + } + } + return newfiles; +} + +void create_manifest_delta(int oldversion, int newversion, char *module) +{ + char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL; + char *conf = config_output_dir(); + int ret; + + if (!conf) { + assert(0); + } + + string_or_die(&newfile, "%s/%i/Manifest.%s", conf, newversion, module); + string_or_die(&original, "%s/%i/Manifest.%s", conf, oldversion, module); + + if (access(newfile, R_OK) != 0 || access(original, R_OK) != 0) { + goto exit; + } + + string_or_die(&outfile, "%s/%i/Manifest-%s-delta-from-%i", conf, newversion, module, oldversion); + string_or_die(&dotfile, "%s/%i/.Manifest-%s-delta-from-%i", conf, newversion, module, oldversion); + + ret = xattrs_compare(original, newfile); + if (ret != 0) { + LOG(NULL, "xattrs have changed, don't create diff ", "%s", newfile); + sleep(1); + } else if (make_bsdiff_delta(original, newfile, dotfile, 0) == 0) { + if (rename(dotfile, outfile) != 0) { + if (errno == ENOENT) { + LOG(NULL, "dotfile:", " %s does not exist", dotfile); + } + LOG(NULL, "Failed to rename", ""); + } + if (!signature_sign(outfile)) { + fprintf(stderr, "Creating signature for '%s' failed\n", outfile); + } + } else { + sleep(1); /* we raced. whatever. sleep for a bit to get the other guy to make progress */ + } + +exit: + free(conf); + free(newfile); + free(original); + free(outfile); + free(dotfile); +} + +void create_manifest_deltas(struct manifest *manifest, GList *last_versions_list) +{ + GList *item; + int prev_version; + + LOG(NULL, "Creating manifest deltas", "%d %s", manifest->version, manifest->component); + /* Create manifest deltas from the previous manifests */ + item = g_list_first(last_versions_list); + while (item) { + prev_version = GPOINTER_TO_INT(item->data); + item = g_list_next(item); + create_manifest_delta(prev_version, manifest->version, manifest->component); + } + LOG(NULL, "Done creating manifest deltas", ""); +} + +/* adds a struct file for a submanifest and also adds it to the proper list */ +void nest_manifest(struct manifest *parent, struct manifest *sub) +{ + struct file *file; + + file = calloc(1, sizeof(struct file)); + if (file == NULL) { + assert(0); + } + + file->last_change = sub->version; + hash_set_zeros(file->hash); + file->is_manifest = 1; + file->filename = strdup(sub->component); + + parent->manifests = g_list_prepend(parent->manifests, file); + parent->submanifests = g_list_prepend(parent->submanifests, sub); + parent->count++; +} + + +/* adds a struct file for a submanifest and also adds it to the proper list */ +void nest_manifest_file(struct manifest *parent, struct file *file) +{ + struct manifest *sub; + + sub = manifest_from_file(file->last_change, file->filename); + + parent->manifests = g_list_prepend(parent->manifests, file); + parent->submanifests = g_list_prepend(parent->submanifests, sub); + parent->count++; + + + LOG(file, "Nest manifest file", "%s", file->filename); +} + + +int manifest_subversion(struct manifest *parent, char *group) +{ + GList *list; + struct file *file; + + list = parent->manifests; + while (list) { + file = list->data; + list = g_list_next(list); + + if (strcmp(file->filename, group) == 0) { + return file->last_change; + } + } + LOG(NULL, "No sub package found, returning 0", ""); + return 0; +} + +static void maximize_version_manifests(struct manifest *m1, struct manifest *m2) +{ + GList *list1, *list2; + struct file *file1, *file2; + + if (!m1) { + printf("Maximizing manifests failed: No m1 manifest!\n"); + return; + } + + LOG(NULL, "Maximizing", "%s to full", m1->component); + + if (!m2) { + printf("Maximizing manifests failed: No m2 manifest!\n"); + return; + } + + m1->files = g_list_sort(m1->files, file_sort_filename); + m2->files = g_list_sort(m2->files, file_sort_filename); + + list1 = g_list_first(m1->files); + list2 = g_list_first(m2->files); + + while (list1 && list2) { + int ret; + file1 = list1->data; + file2 = list2->data; + + ret = strcmp(file1->filename, file2->filename); + if (ret == 0) { + if (file1->last_change > file2->last_change) { + LOG(file1, "Update", "Moving %s to version %i", file1->filename, file1->last_change); + file2->last_change = file1->last_change; + } + list1 = g_list_next(list1); + list2 = g_list_next(list2); + continue; + } + + if (ret < 0) { + list1 = g_list_next(list1); + continue; + } + list2 = g_list_next(list2); + } +} + + +/* + * We need this in case files move modules; + * the "full" list must be the maximum of all the versions of + * all the modules for things to work out. + */ +void maximize_to_full(struct manifest *MoM, struct manifest *full) +{ + struct manifest *sub; + GList *item; + + item = MoM->submanifests; + while (item) { + sub = item->data; + item = g_list_next(item); + maximize_version_manifests(sub, full); + } +} + +void recurse_manifest(struct manifest *manifest) +{ + GList *list; + struct file *file; + struct manifest *sub; + + list = manifest->manifests; + while (list) { + file = list->data; + list = g_list_next(list); + + if (1) { + int version2; + version2 = file->last_change; + + sub = manifest_from_file(version2, file->filename); + manifest->submanifests = g_list_prepend(manifest->submanifests, sub); + } + } +} + +void consolidate_submanifests(struct manifest *manifest) +{ + GList *list, *next; + struct manifest *sub; + struct file *file1, *file2; + + /* Create a consolidated, sorted list of files from all of the + * manifests' lists of files. */ + list = g_list_first(manifest->submanifests); + while (list) { + sub = list->data; + list = g_list_next(list); + if (!sub) { + continue; + } + manifest->files = g_list_concat(sub->files, manifest->files); + sub->files = NULL; + } + manifest->files = g_list_sort(manifest->files, file_sort_filename); + + /* Two pointers ("list" and "next") traverse the consolidated, filename sorted + * GList of files. The "list" pointer is marched forward through the + * GList as long as it and the g_list_next(list) (aka "next") do not point + * to two objects with the same filename. If the name is the same, then + * "list" and "next" point to the first and second in a series of perhaps + * many objects referring to the same filename. As we determine which file out + * of multiples to keep in our consolidated, deduplicated, filename sorted list + * there are Manifest invariants to maintain. The following table shows the + * associated decision matrix. Note that "file" may be a file, directory or + * symlink. + * + * | File 2: + * | A' B' C' D' + * File 1: |------------------------ + * A | - | 2 | 2 | 2 | + * B | 1 | - | 2 | 2 | + * C | 1 | 1 | - | X | + * D | 1 | 1 | X | X | + * + * State for file1 {A,B,C,D} + * for file2 {A',B',C',D'} + * A: is_deleted && !is_rename + * B: is_deleted && is_rename + * C: !is_deleted && (file1->hash == file2->hash) + * D: !is_deleted && (file1->hash != file2->hash) + * + * Action + * -: Don't Care - choose/remove either file + * X: Error State - remove both files, LOG error + * 1: choose file1 - remove file2 + * 2: choose file2 - remove file1 + * + * NOTE: the code below could be rewritten to be more "efficient", but clarity + * and concreteness here are of utmost importance if we are to correctly + * maintain the installed system's state in the filesystem across updates + */ + list = g_list_first(manifest->files); + while (list) { + next = g_list_next(list); + if (next == NULL) { + break; + } + file1 = list->data; + file2 = next->data; + + if (strcmp(file1->filename, file2->filename)) { + list = next; + continue; + } /* from here on, file1 and file2 have a filename match */ + + /* (case 1) A' : choose file1 */ + if (file2->is_deleted && !file2->is_rename) { + manifest->files = g_list_delete_link(manifest->files, next); + continue; + } + /* (case 2) A : choose file2 */ + if (file1->is_deleted && !file1->is_rename) { + manifest->files = g_list_delete_link(manifest->files, list); + list = next; + continue; + } + /* (case 3) B' AND NOT A : choose file 1*/ + if (file2->is_deleted && file2->is_rename) { /* && !(file1->is_deleted && !file1->is_rename) */ + manifest->files = g_list_delete_link(manifest->files, next); + continue; + } + + /* (case 4) B AND NOT (A' OR B') : choose file2 */ + if (file1->is_deleted && file1->is_rename) { /* && !(file2->is_deleted) */ + manifest->files = g_list_delete_link(manifest->files, list); + list = next; + continue; + } + + /* (case 5) C and C' : choose file1 */ + if (!file1->is_deleted && !file2->is_deleted && hash_compare(file1->hash, file2->hash)) { + manifest->files = g_list_delete_link(manifest->files, next); + continue; + } + + /* (case 6) all others constitute errors */ + LOG(NULL, "unhandled filename pair: file1", "%s %s (%d), file2 %s %s (%d)", + file1->filename, file1->hash, file1->last_change, + file1->filename, file2->hash, file2->last_change); + manifest->files = g_list_delete_link(manifest->files, list); + list = g_list_next(next); + manifest->files = g_list_delete_link(manifest->files, next); + printf("CONFLICT IN MANIFESTS\n"); + + } +} + + +int previous_version_manifest(struct manifest *mom, char *name) +{ + GList *list; + struct manifest *sub; + + list = g_list_first(mom->submanifests); + while (list) { + sub = list->data; + list = g_list_next(list); + if (!sub) { + continue; + } + + if (strcmp(sub->component, name) == 0) { + return sub->prevversion; + } + } + return 0; +} + diff --git a/src/pack.c b/src/pack.c new file mode 100644 index 0000000..42978ae --- /dev/null +++ b/src/pack.c @@ -0,0 +1,497 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * Regis Merlino + * tkeel + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +static void empty_pack_stage(int full, int version, char *module) +{ + char *cmd; + char *path; + int ret; + + // clean any stale data (eg: re-run after a failure) + string_or_die(&cmd, "rm -rf %s/%s/%i/", packstage_dir, module, version); + ret = system(cmd); + if (ret) { + fprintf(stderr, "Failed to clean %s/%s/%i\n", + packstage_dir, module, version); + exit(EXIT_FAILURE); + } + free(cmd); + + if (!full) { + // (re)create module/version/{delta,staged} + string_or_die(&path, "%s/%s/%i/delta", packstage_dir, module, version); + g_mkdir_with_parents(path, S_IRWXU | S_IRWXG); + free(path); + string_or_die(&path, "%s/%s/%i/staged", packstage_dir, module, version); + g_mkdir_with_parents(path, S_IRWXU | S_IRWXG); + free(path); + } +} + +/* + * for very small files (eh.. lets make that all), we untar them for pack purposes + */ +static void explode_pack_stage(int version, char *module) +{ + DIR *dir; + struct dirent *entry; + struct stat buf; + char *path; + + string_or_die(&path, "%s/%s/%i/staged", packstage_dir, module, version); + g_mkdir_with_parents(path, S_IRWXU | S_IRWXG); + dir = opendir(path); + if (!dir) { + fprintf(stderr, "There are problems accessing %s, exiting\n", path); + exit(EXIT_FAILURE); + } + free(path); + while (1) { + char *path, *tar; + int ret; + + entry = readdir(dir); + if (!entry) { + break; + } + + if ((strcmp(entry->d_name, ".") == 0) || + (strcmp(entry->d_name, "..") == 0) || + (strstr(entry->d_name, ".tar") == NULL)) { + continue; + } + + string_or_die(&path, "%s/%s/%i/staged/%s", + packstage_dir, module, version, entry->d_name); + ret = stat(path, &buf); + if (ret) { + free(path); + continue; + } + + /* + * initially, we only untar'd small files and broke out of the loop here. + * However, it turns out that even for larger files it's worth uncompressing, + * the resulting pack is slightly smaller, and in addition, we're saving CPU + * time on the client... + */ + string_or_die(&tar, "tar --directory=%s/%s/%i/staged --warning=no-timestamp " + TAR_PERM_ATTR_ARGS " -axf %s", packstage_dir, module, version, path); + ret = system(tar); + if (!ret) { + unlink(path); + } + free(tar); + free(path); + } + closedir(dir); +} + +static void prepare_pack(struct packdata *pack) +{ + struct manifest *manifest; + + pack->fullcount = 0; + + manifest = manifest_from_file(pack->from, pack->module); + if (!manifest || ((manifest->count == 0) && (manifest->version > 0))) { + free(manifest); + return; + } + + pack->end_manifest = manifest_from_file(pack->to, pack->module); + + empty_pack_stage(0, pack->from, pack->module); + + match_manifests(manifest, pack->end_manifest); + + link_renames(pack->end_manifest->files, manifest); +} + +static void make_pack_full_files(struct packdata *pack) +{ + GList *item; + struct file *file; + int ret; + + LOG(NULL, "starting pack full file creation", "%s: %d to %d", + pack->module, pack->from, pack->to); + + /* full files pack: */ + item = g_list_first(pack->end_manifest->files); + while (item) { + file = item->data; + item = g_list_next(item); + if ((!file->peer || file->peer->is_deleted) && !file->is_deleted && !file->rename_peer) { + char *from, *to; + /* hardlink each file that is in but not in */ + string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash); + string_or_die(&to, "%s/%s/%i/staged/%s.tar", packstage_dir, pack->module, pack->from, file->hash); + ret = link(from, to); + if (ret) { + if (errno != EEXIST) { + LOG(NULL, "Failure to link", "%s to %s (%s) %i", from, to, strerror(errno), errno); + } + } else { + pack->fullcount++; + } + + free(from); + free(to); + } + } + + LOG(NULL, "finished pack full file creation", "%s: %d to %d", + pack->module, pack->from, pack->to); +} + +static int find_file_in_list(GList *files, struct file *file) +{ + GList *item; + struct file *item_file; + + item = g_list_first(files); + + while (item) { + item_file = item->data; + item = g_list_next(item); + + if (hash_compare(item_file->hash, file->hash) && + (item_file->last_change == file->last_change) && + (item_file->peer->last_change == file->peer->last_change)) { + LOG(NULL, "Found a duplicate delta", "%d %d %s %s", file->peer->last_change, file->last_change, file->hash, file->filename); + return TRUE; + } + } + + return FALSE; +} + +static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack) +{ + GList *item; + struct file *file; + char *from; + struct stat stat_delta; + int ret; + + if (!pack->end_manifest) { + return files; + } + + item = g_list_first(pack->end_manifest->files); + + while (item) { + file = item->data; + item = g_list_next(item); + + if ((file->last_change <= pack->from) || + (!file->peer) || + (!file->is_file && !file->is_dir && !file->is_link)) { + continue; + } + + string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change, + file->peer->last_change, file->last_change, file->hash); + + ret = stat(from, &stat_delta); + if (ret && !find_file_in_list(files, file)) { + files = g_list_prepend(files, file); + } + + free(from); + } + + return files; +} + +static void create_delta(gpointer data, __unused__ gpointer user_data) +{ + struct file *file = data; + + /* if the file was not found in the from version, skip delta creation */ + if (file->peer) { + __create_delta(file, file->peer->last_change); + } +} + +static void make_pack_deltas(GList *files) +{ + GThreadPool *threadpool; + GList *item; + struct file *file; + int ret; + GError *err = NULL; + + LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN)); + threadpool = g_thread_pool_new(create_delta, NULL, + sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL); + + item = g_list_first(files); + while (item) { + file = item->data; + item = g_list_next(item); + + ret = g_thread_pool_push(threadpool, file, &err); + if (ret == FALSE){ + // intentionally non-fatal + fprintf(stderr, "GThread create_delta push error\n"); + fprintf(stderr, "%s\n", err->message); + return; + } + } + + g_thread_pool_free(threadpool, FALSE, TRUE); +} + +/* Returns 0 == success, -1 == failure */ +static int write_pack_signature(struct packdata *pack) +{ + char *filename = NULL; + int ret = -1; + + string_or_die(&filename, "%s/%i/pack-%s-from-%i.tar", + staging_dir, pack->to, pack->module, pack->from); + if (!signature_sign(filename)) { + fprintf(stderr, "Creating signature for '%s' failed\n", filename); + goto exit; + } + ret = 0; +exit: + free(filename); + return ret; +} + +/* Returns 0 == success, other == failure */ +static int make_final_pack(struct packdata *pack) +{ + GList *item; + struct file *file; + int ret; + char *tar; + double penalty; + + LOG(NULL, "make_final_pack", "%s: %i to %i", pack->module, pack->from, pack->to); + + item = g_list_first(pack->end_manifest->files); + + while (item) { + char *from, *to, *tarfrom, *tarto; + struct stat stat_delta, stat_tar; + + file = item->data; + item = g_list_next(item); + + if ((file->last_change <= pack->from) || + (!file->peer) || + (!file->is_file && !file->is_dir && !file->is_link)) { + continue; + } + + /* for each file changed since */ + /* locate delta, check if the diff it's from is >= */ + string_or_die(&from, "%s/%i/delta/%i-%i-%s", staging_dir, file->last_change, + file->peer->last_change, file->last_change, file->hash); + string_or_die(&to, "%s/%s/%i/delta/%i-%i-%s", packstage_dir, pack->module, + pack->from, file->peer->last_change, file->last_change, file->hash); + string_or_die(&tarfrom, "%s/%i/files/%s.tar", staging_dir, + file->last_change, file->hash); + string_or_die(&tarto, "%s/%s/%i/staged/%s.tar", packstage_dir, pack->module, + pack->from, file->hash); + + ret = stat(from, &stat_delta); + if (ret) { + stat_delta.st_size = LONG_MAX; + } + + ret = stat(tarfrom, &stat_tar); + if (ret) { + stat_tar.st_size = LONG_MAX; + } + + if (file->is_dir) { + stat_delta.st_size = LONG_MAX; + } + + if (stat_delta.st_size <= 8) { + stat_delta.st_size = LONG_MAX; + } + + penalty = 1.05 * (double)(stat_delta.st_size); + + /* delta files get a 5% penalty, they are more cpu work on the client */ + if ((penalty < (double)(stat_tar.st_size)) || stat_tar.st_size == 0) { + /* include delta file in pack */ + ret = link(from, to); + if (ret) { + if (errno != EEXIST) { + LOG(NULL, "Failure to link", "%s to %s (%s) %i\n", from, to, strerror(errno), errno); + } + } + } else { + /* include full file in pack */ + ret = link(tarfrom, tarto); + if (ret) { + if (errno != EEXIST) { + LOG(NULL, "Failure to link", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno); + } + } else { + pack->fullcount++; + } + } + + free(from); + free(to); + free(tarfrom); + free(tarto); + } + + if (pack->fullcount > 0) { + /* untar all files for smaller size and less cpu use on client */ + explode_pack_stage(pack->from, pack->module); + } + + /* now... link in the Manifest pack */ + if (pack->from != 0) { + char *from, *to; + struct stat st; + + string_or_die(&from, "%s/%i/Manifest-%s-delta-from-%i", + staging_dir, pack->to, pack->module, pack->from); + + ret = stat(from, &st); + if (ret) { + LOG(NULL, "Making extra manifest delta", "%s: %i->%i", pack->module, pack->from, pack->to); + create_manifest_delta(pack->from, pack->to, pack->module); + } + + + string_or_die(&to, "%s/%s/%i/Manifest-%s-delta-from-%i", + packstage_dir, pack->module, pack->from, pack->module, pack->from); + + ret = link(from, to); + if (ret) { + LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from); + } + + free(from); + free(to); + } + + + /* now... link in the MoM Manifest into the base pack */ + if ((pack->from != 0) && (strcmp(pack->module, "os-core") == 0)) { + char *from, *to; + struct stat st; + + string_or_die(&from, "%s/%i/Manifest-%s-delta-from-%i", + staging_dir, pack->to, "MoM", pack->from); + ret = stat(from, &st); + if (ret) { + LOG(NULL, "Making extra manifest delta", "MoM: %i->%i", pack->from, pack->to); + create_manifest_delta(pack->from, pack->to, "MoM"); + } + + string_or_die(&to, "%s/%s/%i/Manifest-%s-delta-from-%i", + packstage_dir, pack->module, pack->from, "MoM", pack->from); + + ret = link(from, to); + if (ret) { + LOG(NULL, "Failed to link", "Manifest-delta-from-%i", pack->from); + } + + free(from); + free(to); + } + + + /* tar the staging directory up */ + LOG(NULL, "starting tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to); + string_or_die(&tar, "tar " TAR_PERM_ATTR_ARGS " --directory=%s/%s/%i/ " + "--numeric-owner -Jcf %s/%i/pack-%s-from-%i.tar delta staged", + packstage_dir, pack->module, pack->from, staging_dir, pack->to, pack->module, pack->from); + ret = system(tar); + free(tar); + LOG(NULL, "finished tar for pack", "%s: %i to %i", pack->module, pack->from, pack->to); + /* FIXME: this is a hack workaround, needs diagnosed and removed */ + if ((ret != 0) && (ret != 256)) { + fprintf(stderr, "Unexpected return value (%d) creating tar of pack %s from %i to %i\n", + ret, pack->module, pack->from, pack->to); + } else { + /* Write the signature file */ + ret = write_pack_signature(pack); + if (ret != 0) { + fprintf(stderr, "Failure creating signature of pack %s from %i to %i\n", + pack->module, pack->from, pack->to); + } + } + + /* and clean up */ + free_manifest(pack->end_manifest); + empty_pack_stage(1, pack->from, pack->module); + + LOG(NULL, "pack complete", "%s: %i to %i", pack->module, pack->from, pack->to); + return ret; +} + + +/* Returns 0 == success, -1 == failure */ +int make_pack(struct packdata *pack) +{ + GList *delta_list = NULL; + + /* step 1: prepare pack */ + prepare_pack(pack); + + /* step 2: consolidate delta list & create all delta files*/ + delta_list = consolidate_packs_delta_files(delta_list, pack); + make_pack_deltas(delta_list); + g_list_free(delta_list); + + /* step 3: complete pack creation */ + if (!pack->end_manifest) { + return 0; + } + make_pack_full_files(pack); + make_final_pack(pack); + + return 0; +} diff --git a/src/rename.c b/src/rename.c new file mode 100644 index 0000000..acc57e4 --- /dev/null +++ b/src/rename.c @@ -0,0 +1,435 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" + + +#include + +static magic_t mcookie; + + +double rename_score(struct file *old, struct file *new) +{ + double score = 0.0; + char *c; + int io, in; + + + /* if we have the same hash... sounds like a winner */ + if (hash_compare(old->hash, new->hash)) { + score += 400; + } + + /* points for being in the same directory */ + if (strcmp(old->dirname, new->dirname) == 0) { + score += 10; + } + + /* points for the basename sharing a beginning */ + c = strchr(old->basename, '.'); + if (c) { + int len; + len = c - old->basename; + if (strncmp(old->basename, new->basename, len) == 0) { + score += len; + } + } + + /* extra points for having an identical basename */ + if (strcmp(old->basename, new->basename) == 0) { + score += 35; + } + + /* points for being the same if you remove non-letters */ + if (strcmp(old->alpha_only_filename, new->alpha_only_filename) == 0) { + score += 50; + } + + /* walking the directory backwards, for each directory that's the same, five points */ + io = strlen(old->dirname) - 1; + in = strlen(new->dirname) - 1; + while (io >= 0 && in >= 0 && old->dirname[io] == new->dirname[in]) { + if (old->dirname[io] == '/') { + score += 5; + } + io--; + in--; + } + + /* if both start with /boot/vmlinuz give it a boost; this is a local hack due to vmlinuz being very short */ + if (strncmp(old->filename, "/boot/vmlinuz", 13) == 0 && strncmp(new->filename, "/boot/vmlinuz", 13) == 0) { + score += 80; + } + + /* if ELF, points for sharing the same soname to the first dot */ + + /* negative points for not being within 25%+/-1Kb of the same file size */ + if (old->stat.st_size > ((new->stat.st_size * 1.25) + 1024)) { + score -= 30; + } + if (old->stat.st_size < ((new->stat.st_size * 0.75) - 1024)) { + score -= 30; + } + /* zero sized stuff.. never a good idea */ + if (old->stat.st_size == 0 || new->stat.st_size == 0) { + score = -100; + return score; + } else { + /* size ratio makes a good tiebreaker for sorting; the more the same the more likely to be a match */ + double s; + s = old->stat.st_size / new->stat.st_size; + if (s > 1) { + s = new->stat.st_size / old->stat.st_size; + } + score += s; + } + /* negative points for not having the same 'file' type */ + + if (old->filetype && new->filetype && strcmp(old->filetype, new->filetype) != 0) { + score -= 60; + } + + return score; +} + +static void precompute_file_data(struct manifest *manifest, struct file *file, int old_rename, GList *last_versions_list) +{ + GList *item; + char *c1, *c2; + char *filename = NULL; + int last_change; + struct stat buf; + + /* fill in the filename-minus-the-numbers field */ + file->alpha_only_filename = calloc(strlen(file->filename) + 1, sizeof(char)); + + c1 = file->filename; + c2 = file->alpha_only_filename; + while (*c1) { + while (*c1 && !isalpha(*c1)) + c1++; + if (!*c1) { + break; + } + if (c2 == NULL) { + break; + } + *c2 = *c1; + c1++; + c2++; + } + + if (manifest) { + string_or_die(&filename, "%s/%i/%s/%s", image_dir, manifest->version, manifest->component, file->filename); + } else if (old_rename) { + item = g_list_first(last_versions_list); + while (item) { + last_change = GPOINTER_TO_INT(item->data); + item = g_list_next(item); + + free(filename); + string_or_die(&filename, "%s/%i/full/%s", image_dir, last_change, file->filename); + if (!lstat(filename, &buf)) { + break; + } + } + } else { + string_or_die(&filename, "%s/%i/full/%s", image_dir, file->last_change, file->filename); + } + + /* make sure file->stat.st_size is valid */ + if (file->stat.st_size == 0) { + int ret; + + if (filename == NULL) { + printf("filename is null...impossible to stat\n"); + assert(0); + } + ret = lstat(filename, &buf); + if (!ret) { + file->stat.st_size = buf.st_size; + } else { + printf("Stat failure on %s\n", filename); + } + } + + c1 = (char *) magic_file(mcookie, filename); + if (c1) { + char *c2; + file->filetype = strdup(c1); + c2 = strstr(file->filetype, "not stripped"); + if (c2) { + *c2 = 0; + } + c2 = strstr(file->filetype, "stripped"); + if (c2) { + *c2 = 0; + } + } else { + LOG(file, "Cannot find file type", "%s", filename); + } + + free(filename); + + file->basename = strdup(basename(file->filename)); + file->dirname = strdup(dirname(file->filename)); + +} + + +int file_sort_score(gconstpointer a, gconstpointer b) +{ + struct file *A, *B; + + A = (struct file *) a; + B = (struct file *) b; + + if (A->rename_score > B->rename_score) { + return -1; + } + if (A->rename_score < B->rename_score) { + return 1; + } + + return 0; +} + +static void score_file(GList *deleted_files, struct file *file) +{ + GList *list2; + struct file *file2; + + file->rename_score = -100; + + list2 = g_list_first(deleted_files); + while (list2) { + double score; + file2 = list2->data; + list2 = g_list_next(list2); + + + assert(file2); + assert(file2->peer); + + score = rename_score(file2->peer, file); + if (score > file->rename_score) { + file->rename_score = score; + file->rename_peer = file2; + } + + } +} + +void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list) +{ + GList *new_files = NULL; + GList *deleted_files = NULL; + + GList *list; + struct file *file; + int old_rename = 0; + + if (last_change != manifest->version) { + old_rename = 1; + } + + if (mcookie == NULL) { + mcookie = magic_open(MAGIC_NO_CHECK_COMPRESS); + magic_load(mcookie, NULL); + } + + /* make a list of new files, no peer */ + list = g_list_first(manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + if ((file->last_change != manifest->version) || + (file->is_deleted) || + (!file->is_file) || + (file->peer)) { + continue; + } + + new_files = g_list_prepend(new_files, file); + precompute_file_data(manifest, file, old_rename, last_versions_list); + } + + /* if there are no new files, we're not having any renames -- early exit */ + if (!new_files) { + LOG(NULL, "No new files, no rename detection", "%s", manifest->component); + return; + } + /* make a list of newly deleted files that have a peer */ + + list = g_list_first(manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + if ((!file->is_deleted) || + (!file->peer) || + (file->last_change != last_change) || + (file->peer->is_dir || file->peer->is_link)) { + continue; + } + + deleted_files = g_list_prepend(deleted_files, file); + precompute_file_data(NULL, file->peer, old_rename, last_versions_list); + } + + /* nothing got deleted --> no renames --> early exit */ + if (!deleted_files) { + LOG(NULL, "No deleted files, no rename detection", "%s", manifest->component); + g_list_free(new_files); + return; + } + + /* for each new file, find the deleted file with the highest score, and store the score */ + list = g_list_first(new_files); + while (list) { + file = list->data; + list = g_list_next(list); + score_file(deleted_files, file); + } + + /* sort all new files by score */ + + new_files = g_list_sort(new_files, file_sort_score); + + /* pick the top score, check if the score is still valid, if not, recompute the score and resort */ + + while (new_files) { + file = new_files->data; + + if (file->rename_score < 15.0 || file->rename_peer == NULL) { + new_files = g_list_delete_link(new_files, new_files); + if (file->rename_peer) { + LOG(NULL, "Rename not done due to insufficient high score", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); + } + continue; + } + + if (file->rename_peer->rename_peer != NULL) { + /* the candidate peer got already taken by another file! */ + LOG(NULL, "Rename not done due to target already taken", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); + file->rename_peer = NULL; + file->rename_score = -100; + score_file(deleted_files, file); + new_files = g_list_sort(new_files, file_sort_score); + continue; + } + + /* if valid and score is high enough, make the link by setting the flag and storing the hash */ + + LOG(NULL, "Rename detected!", "%s -> %s score %4.1f", file->rename_peer->filename, file->filename, file->rename_score); + + file->rename_peer->rename_peer = file; + /* must delete the file from the deleted list */ + deleted_files = g_list_remove(deleted_files, file->rename_peer); + hash_assign(file->hash, file->rename_peer->hash); + file->is_rename = 1; + file->rename_peer->is_rename = 1; + + new_files = g_list_delete_link(new_files, new_files); + + } /* lather, rinse, repeat until all files have a target */ + + /* free memory */ + g_list_free(new_files); + g_list_free(deleted_files); +} + +static int file_found_in_older_manifest(struct manifest *from_manifest, struct file *searched_file) +{ + GList *list; + struct file *file; + + list = g_list_first(from_manifest->files); + while (list) { + file = list->data; + list = g_list_next(list); + + if (file->is_deleted) { + continue; + } + if (!strcmp(file->filename, searched_file->filename)) { + return 1; + } + } + + return 0; +} + +void link_renames(GList *newfiles, struct manifest *from_manifest) +{ + GList *list1, *list2; + GList *targets; + struct file *file1, *file2; + + targets = newfiles = g_list_sort(newfiles, file_sort_version); + + list1 = g_list_first(newfiles); + + /* todo: sort newfiles and targets by hash */ + + while (list1) { + file1 = list1->data; + list1 = g_list_next(list1); + + if ((file1->peer || !file1->is_rename) || + (file1->is_deleted)) { + continue; + } + /* now, file1 is the new file that got renamed. time to search the rename targets */ + list2 = g_list_first(targets); + while (list2) { + file2 = list2->data; + list2 = g_list_next(list2); + + if ((!file2->peer || !file2->is_rename) || + (!file2->is_deleted) || + (!file_found_in_older_manifest(from_manifest, file2))) { + continue; + } + if (hash_compare(file2->hash, file1->hash)) { + file1->rename_peer = file2->peer; + file1->peer = file2->peer; + file2->peer->rename_peer = file1; + list2 = NULL; + } + } + } + free(from_manifest); +} diff --git a/src/signature.c b/src/signature.c new file mode 100644 index 0000000..d1663f5 --- /dev/null +++ b/src/signature.c @@ -0,0 +1,171 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Tom Keel + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" + +static char *make_filename(const char *, const char *, const char *); + +static const char *CMD_FMT = "openssl smime -sign -in %s -binary " + "-out %s.signed -outform PEM -md sha256 -inkey %s -signer %s " + "-certfile %s -passin file:%s"; + +static char *leaf_key = NULL; +static char *leaf_cert = NULL; +static char *ca_chain_cert = NULL; +static char *passphrase = NULL; + +static bool initialized = false; + +/* + * Initialize this module. + * @return true <=> success + */ +bool signature_initialize(void) +{ + if (!enable_signing) { + return true; + } + + char *cdir; + char *pphr; + struct stat s; + + if (initialized) { + return true; + } + cdir = getenv("SWUPD_CERTS_DIR"); + if (cdir == NULL || cdir[0] == '\0') { + printf("No certificates directory specified\n"); + goto err; + } + if (stat(cdir, &s)) { + printf("Can't stat certificates directory '%s' (%s)\n", cdir, + strerror(errno)); + goto err; + } + leaf_key = make_filename(cdir, "LEAF_KEY", "leaf key"); + if (leaf_key == NULL) { + goto err; + } + leaf_cert = make_filename(cdir, "LEAF_CERT", "leaf certificate"); + if (leaf_cert == NULL) { + goto err; + } + ca_chain_cert = make_filename(cdir, "CA_CHAIN_CERT", "CA chain certificate"); + if (ca_chain_cert == NULL) { + goto err; + } + pphr = getenv("PASSPHRASE"); + if (pphr == NULL || (passphrase = strdup(pphr)) == NULL) { + goto err; + } + if (stat(passphrase, &s)) { + printf("Can't stat '%s' (%s)\n", passphrase, + strerror(errno)); + goto err; + } + initialized = true; + return true; +err: + signature_terminate(); + return false; +} + +/* Make filename from dir name and env variable containing basename */ +static char *make_filename(const char *dir, const char *env, const char *desc) +{ + char *fn = getenv(env); + char *result = NULL; + struct stat s; + + if (fn == NULL || fn[0] == '\0') { + printf("No %s file specified\n", desc); + return NULL; + } + string_or_die(&result, "%s/%s", dir, fn); + if (stat(result, &s)) { + printf("Can't stat %s '%s' (%s)\n", desc, result, strerror(errno)); + free(result); + return NULL; + } + return result; +} + +/* + * Terminate this module, free resources. + */ +void signature_terminate(void) +{ + if (!enable_signing) { + return; + } + + free(leaf_key); + free(leaf_cert); + free(ca_chain_cert); + free(passphrase); + + leaf_key = NULL; + leaf_cert = NULL; + ca_chain_cert = NULL; + passphrase = NULL; + + initialized = false; +} + +/* + * Write the signature file corresponding to the given data file. + * The name of the signature file is the name of the data file with suffix + * ".signed" appended. + */ +bool signature_sign(const char *filename) +{ + char *cmd = NULL; + int status; + + if (!enable_signing) { + return true; + } + + if (!initialized) { + return false; + } + string_or_die(&cmd, CMD_FMT, filename, filename, leaf_key, leaf_cert, + ca_chain_cert, passphrase); + status = system(cmd); + if (status) { + printf("Bad status %d from signing command:%s\n", status, cmd); + } + free(cmd); + return status == 0; +} diff --git a/src/stats.c b/src/stats.c new file mode 100644 index 0000000..06fe87b --- /dev/null +++ b/src/stats.c @@ -0,0 +1,102 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Tim Pepper + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + + + + +static int new_files; +static int deleted_files; +static int changed_files; +static int delta_miss; +static int delta_hit; + + +void account_new_file(void) +{ + new_files++; +} + +void account_deleted_file(void) +{ + deleted_files++; +} + +void account_changed_file(void) +{ + changed_files++; +} + +void account_delta_hit(void) +{ + delta_hit++; +} + + +void account_delta_miss(void) +{ + delta_miss++; +} + +int have_delta_files(void) +{ + if (changed_files) { + return 1; + } + return 0; +} + +int have_new_files(void) +{ + if (new_files) { + return 1; + } + return 0; +} + + + +void print_statistics(int version1, int version2) +{ + printf("\n"); + printf("Statistics for going from version %i to version %i\n", version1, version2); + printf(" changed files : %i\n", changed_files); + printf(" new files : %i\n", new_files); + printf(" deleted files : %i\n", deleted_files); + printf("\n"); +} + +void print_delta_statistics(void) +{ + LOG(NULL, "Delta stats", "%i successful delta usages, %i failures", delta_hit, delta_miss); +} diff --git a/src/type_change.c b/src/type_change.c new file mode 100644 index 0000000..7172163 --- /dev/null +++ b/src/type_change.c @@ -0,0 +1,112 @@ +/* + * Software Updater - server side + * + * Copyright © 2014-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Tom Keel + * + */ + +#define _GNU_SOURCE +#include +#include +#include + +#include + +#include "swupd.h" + +static char file_type(struct file *file) +{ + if (file->is_file) { + return 'F'; + } + if (file->is_dir) { + return 'D'; + } + if (file->is_link) { + return 'L'; + } + if (file->is_manifest) { + return 'M'; + } + + // The concept of file type must have been extended... + fprintf(stderr, "Unknown type for %s .. exiting\n", file->filename); + exit(EXIT_FAILURE); + return 0; +} + +static bool type_has_changed(struct file *file) +{ + char type1, type2; + + if (file == NULL || file->peer == NULL) { + // FIXME: Should we assert(0) here? Or not test at all? + return false; + } + + if (file->peer->is_deleted) { + return false; + } + + if (file->is_deleted) { + return false; + } + + type1 = file_type(file->peer); + type2 = file_type(file); + + if (type1 == type2) { + return false; + } + + LOG(file, "Type change", "%c to %c", type1, type2); + fprintf(stderr, "Type change from %c to %c for %s\n", type1, type2, + file->filename); + + if (((type1 == 'F') && (type2 == 'L')) || + ((type1 == 'F') && (type2 == 'D')) || + ((type1 == 'L') && (type2 == 'F')) || + ((type1 == 'L') && (type2 == 'D'))) { + /* 1) file to symlink is an OK transition + * 2) file to directory is an OK transition + * 3) symlink to file is an OK transition + * 4) symlink to directory is an OK transition + */ + return false; + } + + return true; +} + +void type_change_detection(struct manifest *manifest) +{ + GList *l; + int n = 0; + + for (l = g_list_first(manifest->files); l; l = g_list_next(l)) { + if (type_has_changed(l->data)) { + n++; + } + } + + if (n != 0) { + printf("Detected %d file type changes in component %s.. exiting\n", + n, manifest->component); + exit(EXIT_FAILURE); + } +} diff --git a/src/versions.c b/src/versions.c new file mode 100644 index 0000000..635554d --- /dev/null +++ b/src/versions.c @@ -0,0 +1,297 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Sebastien Boeuf + * + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" + +int current_version; + + +void read_current_version(char *filename) +{ + FILE *file; + int v = 0; + char *fullfile = NULL; + char *conf; + + conf = config_image_base(); + + + string_or_die(&fullfile, "%s/%s", conf, filename); + + file = fopen(fullfile, "rm"); + + free(conf); + + if (!file) { + fprintf(stderr, "Cannot read current version, failed to open %s (%s)\n", fullfile, strerror(errno)); + free(fullfile); + return; + } + + if (fscanf(file, "%i", &v) < 0) { + /* not even a single int there --> make sure to return 0 */ + fprintf(stderr, "Version file %s does not have a number in it. Setting version to 0\n", fullfile); + } + + fclose(file); + free(fullfile); + + current_version = v; +} + +void write_new_version(char *filename, int version) +{ + FILE *file; + char *fullfile = NULL; + char *conf; + + conf = config_image_base(); + + + string_or_die(&fullfile, "%s/%s", conf, filename); + + file = fopen(fullfile, "w"); + + free(conf); + + if (!file) { + fprintf(stderr, "Cannot write new version, failed to open %s (%s)\n", fullfile, strerror(errno)); + free(fullfile); + return; + } + + fprintf(file, "%i\n", version); + + fclose(file); + free(fullfile); +} + +void ensure_version_image_exists(int version) +{ + char *conf; + char *path = NULL; + struct stat buf; + int ret; + + conf = config_image_base(); + + string_or_die(&path, "%s/%i", conf, version); + + free(conf); + + ret = stat(path, &buf); + if (ret < 0) { + fprintf(stderr, "Failed to stat image directory %s (%s).. exiting\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (!S_ISDIR(buf.st_mode)) { + fprintf(stderr, "Assumed image directory %s is not a directory.. exiting\n", path); + exit(EXIT_FAILURE); + } + + free(path); +} + +void write_cookiecrumbs_to_download_area(int version) +{ + char *conf; + char *cmd; + + conf = config_output_dir(); + if (conf == NULL) { + assert(0); + } + + string_or_die(&cmd, "echo %s > %s/%i/swupd-server-src-version", VERSION, conf, version); + if (system(cmd) != 0) { + assert(0); + } + free(cmd); + + string_or_die(&cmd, "echo %s > %s/%i/format", format_string, conf, version); + if (system(cmd) != 0) { + assert(0); + } + free(cmd); + +/* written from pack_maker.sh */ +#if 0 + FILE *file; + char *fullfile = NULL; + + string_or_die(&cmd, "mkdir -p %s/version/formatstaging/", conf); + if (system(cmd) != 0) { + assert(0); + } + free(cmd); + + string_or_die(&fullfile, "%s/version/formatstaging/latest", conf); + file = fopen(fullfile, "w"); + if (!file) { + fprintf(stderr, "Cannot write new version to download area, failed to open %s (%s)\n", fullfile, strerror(errno)); + free(fullfile); + return; + } + fprintf(file, "%i\n", version); + fclose(file); + free(fullfile); +#endif + free(conf); +} + +static int compare_versions(gconstpointer a, gconstpointer b) +{ + return GPOINTER_TO_INT(b) - GPOINTER_TO_INT(a); +} + +static int get_build_type(int build_num) { + int build_type = build_num % 10; + if (build_type == 0) { + return REGULAR_BUILD; + } else if (build_type == 9) { + return DEV_BUILD; + } else { + return FIX_BUILD; + } +} + +static int get_jump_point(int build_num) { + int jump_point = (build_num / 100) * 100; + if (jump_point == build_num) { + jump_point -= 100; + } + if (jump_point < 0) { + jump_point = 0; + } + return jump_point; +} + +/* create a list of max_versions most recent version numbers, excluding + * next_version */ +GList *get_last_versions_list(int next_version, int max_versions) +{ + DIR *dir; + GList *list = NULL; + GList *cur_item, *next_item; + struct dirent entry; + struct dirent *result; + struct stat stat; + char *filename = NULL; + int idx, build_num, build_type, jump_point; + int jump_point_found; + + dir = opendir(staging_dir); + if (dir == NULL) { + LOG(NULL, "Cannot open directory", "dir_path= %s, strerror= %s", + staging_dir, strerror(errno)); + return NULL; + } + + while (readdir_r(dir, &entry, &result) == 0 && result != NULL) { + if (strspn(entry.d_name, "0123456789") != strlen(entry.d_name)) { + continue; + } + + free(filename); + string_or_die(&filename, "%s/%s", staging_dir, entry.d_name); + + if (lstat(filename, &stat)) { + LOG(NULL, "lstat failed", "path= %s, strerror= %s", + filename, strerror(errno)); + continue; + } + + if (!S_ISDIR(stat.st_mode)) { + continue; + } + + if (atoi(entry.d_name) >= next_version) { + continue; + } + + list = g_list_prepend(list, GINT_TO_POINTER(atoi(entry.d_name))); + } + free(filename); + closedir(dir); + + /* Exit without error if list is empty */ + if (list == NULL) { + return NULL; + } + + /* Sort list top down */ + list = g_list_sort(list, compare_versions); + + /* + * Keep last max_versions regular build versions, + * all fix build versions between, and the most + * recent jump point version + */ + idx = 0; + jump_point = -1; + jump_point_found = 0; + next_item = list = g_list_first(list); + while (next_item) { + cur_item = next_item; + build_num = GPOINTER_TO_INT(cur_item->data); + build_type = get_build_type(build_num); + next_item = g_list_next(cur_item); + if ((idx >= max_versions || build_type == DEV_BUILD) && + build_num != jump_point && build_num != 0) { + list = g_list_delete_link(list, cur_item); + } else if (idx == max_versions - 1 && build_num != 0) { + list = g_list_delete_link(list, cur_item); + idx--; + } + if (build_num == jump_point) { + jump_point_found = 1; + } + if (build_type == REGULAR_BUILD) { + idx++; + if (idx >= max_versions && !jump_point_found) { + if (jump_point < 0) { + jump_point = get_jump_point(build_num); + } else if (build_num < jump_point) { + jump_point = get_jump_point(jump_point); + } + } + } + } + + return list; +} diff --git a/src/xattrs.c b/src/xattrs.c new file mode 100644 index 0000000..2b746d3 --- /dev/null +++ b/src/xattrs.c @@ -0,0 +1,269 @@ +/* + * Software Updater - server side + * + * Copyright © 2014-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Christophe Guiraud + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "swupd.h" +#include "xattrs.h" + +enum xattrs_action_type_t_ { + XATTRS_ACTION_COPY, + XATTRS_ACTION_GET_BLOB +}; + +typedef enum xattrs_action_type_t_ xattrs_action_type_t; + +static int xattr_get_value(const char *path, const char *name, char **blob, + size_t *blob_len, xattrs_action_type_t action) +{ + char *value; + ssize_t len; + + len = lgetxattr(path, name, NULL, 0); + if (len < 0) { + LOG(NULL, "Failed to get x-attribute length", + "%s for file %s: %s", + name, path, strerror(errno)); + return -1; + } + + if (*blob == NULL) { + *blob_len = 0; + } + + /* realloc needed len + 1 in case we need to add final zero + * to ensure consistent blob */ + value = realloc(*blob, *blob_len + len + + (action == XATTRS_ACTION_GET_BLOB ? 1 : 0)); + assert(value); + + *blob = value; + + value = value + *blob_len; + len = lgetxattr(path, name, value, len); + if (len < 0) { + LOG(NULL, "Failed to get x-attribute", + "%s for file %s: %s", + name, path, strerror(errno)); + return -1; + } + + /* in the xattrs system, value is just an arbitrary binary blob. If it + * is a string, it can already be null terminated or not, depending + * on the len that was passed when the attribute was set. So, make + * sure we use a consistent value by adding a final zero if not + * already there. + * note: this must not be done when copying attributes. In this case + * we want to keep them unchanged. It must only be done when + * getting the value blob to use for key computation + */ + if (action == XATTRS_ACTION_GET_BLOB && len && value[len - 1] != 0) { + value[len] = 0; + len++; + } + + *blob_len += len; + + return 0; +} + +static int get_xattr_name_count(const char *names_list, ssize_t len) +{ + int count = 0; + const char *name; + + for (name = names_list; name < (names_list + len); name += strlen(name) + 1) + count++; + + return count; +} + +static int cmp_xattr_name_ptrs(const void *ptr1, const void *ptr2) +{ + return strcmp(*(char * const *)ptr1, *(char * const *)ptr2); +} + +static const char **get_sorted_xattr_name_table(const char *names, int n) +{ + const char **table; + int i; + + table = calloc(1, n * sizeof(char*)); + assert(table); + + for (i = 0; i < n; i++) { + table[i] = names; + names += strlen(names) + 1; + } + + qsort(table, n, sizeof(char*), cmp_xattr_name_ptrs); + + return table; +} + +static void xattrs_do_action(xattrs_action_type_t action, + const char *src_filename, + const char *dst_filename, + char **blob, size_t *blob_len) { + ssize_t len; + char *list; + int ret = 0; + char *value = NULL; + size_t value_len = 0; + const char **sorted_list = NULL; + int count; + int i; + int offset = 0; + + len = llistxattr(src_filename, NULL, 0); + if (len <= 0) { + if (action == XATTRS_ACTION_GET_BLOB) { + *blob_len = 0; + *blob = (void *)0xdeadcafe; + } + return; // no xattrs, this is OK + } + + list = calloc(1, len); + assert(list); + + len = llistxattr(src_filename, list, len); + if (len <= 0) { + if (action == XATTRS_ACTION_GET_BLOB) { + *blob_len = 0; + *blob = (void *)0xdeadcafe; + } + free(list); + return; // no xattrs, this is OK + } + + count = get_xattr_name_count(list, len); + sorted_list = get_sorted_xattr_name_table(list, count); + + if (action == XATTRS_ACTION_GET_BLOB) { + value = calloc(1, len); + assert(value); + + value_len = len; + + for (i = 0; i < count; i++) { + len = strlen(sorted_list[i]) + 1; + memcpy(value + offset, sorted_list[i], len); + offset += len; + } + } + + for (i = 0; i < count; i++) { + /* In the XATTRS_ACTION_COPY case the xattr_get_value(...) calls + * are always performed with 'value = NULL' and 'value_len = 0'. + */ + ret = xattr_get_value(src_filename, sorted_list[i], &value, &value_len, + action); + if (ret < 0) { + free(value); + value_len = 0; + value = NULL; + break; + } + + if (action == XATTRS_ACTION_COPY) { + /* 'value' contains only the attribute value in this + * case. */ + ret = lsetxattr(dst_filename, sorted_list[i], + value, value_len, 0); + free(value); + if (ret < 0) { + LOG(NULL, "Failed to set x-attributes", + "%s: %s", + dst_filename, strerror(errno)); + break; + } + value_len = 0; + value = NULL; + } + } + + if (action == XATTRS_ACTION_GET_BLOB) { + if (value_len != 0) { + *blob_len = value_len; + *blob = value; + } else { + *blob_len = 0; + *blob = (void *)0xdeadcafe; + } + } + + free(list); + free(sorted_list); + + return; +} + +void xattrs_copy(const char *src_filename, const char *dst_filename) +{ + xattrs_do_action(XATTRS_ACTION_COPY, src_filename, dst_filename, + NULL, NULL); +} + +void xattrs_get_blob(const char *filename, char **blob, size_t *blob_len) +{ + xattrs_do_action(XATTRS_ACTION_GET_BLOB, filename, NULL, + blob, blob_len); +} + +int xattrs_compare(const char *filename1, const char *filename2) +{ + char *new_xattrs; + char *old_xattrs; + size_t new_xattrs_len = 0; + size_t old_xattrs_len = 0; + int ret = 0; + + xattrs_get_blob(filename1, &old_xattrs, &old_xattrs_len); + xattrs_get_blob(filename2, &new_xattrs, &new_xattrs_len); + + if ((old_xattrs_len == 0) && (new_xattrs_len == 0)) { + return ret; + } + + if ((old_xattrs_len != new_xattrs_len) || + (memcmp(old_xattrs, new_xattrs, old_xattrs_len) != 0)) { + ret = -1; + } + + if (old_xattrs_len) { + free(old_xattrs); + } + if (new_xattrs_len) { + free(new_xattrs); + } + + return ret; +} diff --git a/template b/template new file mode 100644 index 0000000..54a95d4 --- /dev/null +++ b/template @@ -0,0 +1,33 @@ +/* + * Software Updater - server side + * + * Copyright © 2012-2016 Intel Corporation. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 2 or later of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Authors: + * Arjan van de Ven + * Timothy C. Pepper + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +#include "swupd.h" diff --git a/test/basedir.tar b/test/basedir.tar new file mode 100644 index 0000000..dda665c Binary files /dev/null and b/test/basedir.tar differ diff --git a/test/pack_check.sh b/test/pack_check.sh new file mode 100755 index 0000000..182d0a0 --- /dev/null +++ b/test/pack_check.sh @@ -0,0 +1,58 @@ +# Usage: +# pack_check.sh buildversion [keep] +# pack_check.sh 180 +# pack_check.sh 180 keep +# +# Download the user specified build version's Manifest.os-core and the pack +# from the prior build version. Untar the file and output information on +# the largest files in the pack. +# +# If "keep" is specified the downloaded files will be left in a +# "pack_check" subdirectory for inspection. +# +# Note: this just gives a human a quick sense today of what's big in the pack +# and clues them in to an issue. It needs built into a more detailed automatic +# test is need to insure that the pack has just one of either delta or full +# file for any file changed in the current build version. That then is a +# post-pack-creation automated test that is run as part of the update build. + +VER=$1 +if [ -z "${VER}" ]; then + echo "please provide a build version number, eg: \"180\"" + exit -1 +fi + +TEMPDIR=pack_check/$VER +mkdir -p ${TEMPDIR} +pushd ${TEMPDIR} + +if ! [ -f Manifest.os-core.${VER} ]; then + curl https://download.clearlinux.org/update/${VER}/Manifest.os-core > Manifest.os-core.${VER} +else + echo "skipping download of Manifest.os-core.${VER}, already present" +fi +if ! [ -f pack-os-core-from-$((${VER}-10)).tar ]; then + curl -O https://download.clearlinux.org/update/${VER}/pack-os-core-from-$((${VER}-10)).tar +else + echo "skipping download of pack-os-core-from-$((${VER}-10)).tar, already present" +fi +tar xf pack-os-core-from-$((${VER}-10)).tar + +rm -f biggest_files_names_${VER}.txt + +ls -al --sort=size staged/ | grep -v total | head -n 20 > biggest_files_sizes_${VER}.txt +for hash in $(cat biggest_files_sizes_${VER}.txt | sed -e 's/^\(.*\) \([a-f0-9]*\)$/\2/') +do + grep $hash Manifest.os-core.${VER} | grep -v -e "^.d.r" >> biggest_files_names_${VER}.txt +done + +cat biggest_files_sizes_${VER}.txt | \ + sed -e 's/^\(.*\) \([0-9]*\) [A-F]\(.*\)/\2/' | \ + paste - biggest_files_names_${VER}.txt | column -t + +popd +if [ "$2" != "keep" ]; then + rm -rf ${TEMPDIR} +else + echo "Leaving working files in ${TEMPDIR}" +fi diff --git a/test/test_hardlink.sh b/test/test_hardlink.sh new file mode 100755 index 0000000..577fd31 --- /dev/null +++ b/test/test_hardlink.sh @@ -0,0 +1,89 @@ +#! /bin/bash + +# This creates a bunch of files, all with the same content, and with various +# combinations of xaddrs. It lists them with their inode numbers, then runs +# hardlink on them, and then lists them with inodes again. Since name of each +# file corresponds to is xattrs, you can easily see if expected linking is +# indeed happening. + +TMPDIR=$(mktemp -d /tmp/hardlinktest-XXXXXX) + +makeFile() { + local argC=$# + local fName=${1}-File + shift 1; (( argC = $argC - 1 )) + local nameValPairs=$* + while (( $argC >= 1 )) ; do + if (( $argC == 1 )) ; then + fName=${fName}-${1} + shift 1; (( argC = $argC - 1 )) + else + if [[ -z $2 ]] ; then + fName=${fName}-${1} + else + fName=${fName}-${1}:${2} + fi + shift 2; (( argC = $argC - 2 )) + fi; + done + echo "must have some content else can't set xattrs" > $TMPDIR/$fName + setXattrs $fName $nameValPairs +} + +setXattrs() { + local argC=$# + local fName=${1} + shift 1; (( argC = $argC - 1 )) + local nameValPairs=$* + while (( $argC >= 1 )) ; do + if (( $argC == 1 )) ; then + setfattr -n user.$1 $TMPDIR/$fName + shift 1; (( argC = $argC - 1 )) + else + if [[ -z $2 ]] ; then + setfattr -n user.$1 $TMPDIR/$fName + else + setfattr -n user.$1 -v $2 $TMPDIR/$fName + fi + shift 2; (( argC = $argC - 2 )) + fi; + done +} + +for prefix in A B ; do + makeFile $prefix + + makeFile $prefix XXX XXX + makeFile $prefix XXX + + makeFile $prefix XXX XXX YYY YYY + makeFile $prefix YYY YYY XXX XXX + makeFile $prefix XXX "" YYY YYY + makeFile $prefix XXX XXX YYY + makeFile $prefix XXX "" YYY + makeFile $prefix YYY "" XXX + + makeFile $prefix XXX XXX YYY YYY ZZZ ZZZ + makeFile $prefix YYY YYY XXX XXX ZZZ ZZZ + makeFile $prefix ZZZ ZZZ YYY YYY XXX XXX + + makeFile $prefix XXX XXXXXXXXXX YYY YYYYYYYY ZZZ ZZZ + makeFile $prefix YYY YYYYYYYY XXX XXXXXXXXXX ZZZ ZZZ + makeFile $prefix ZZZ ZZZ YYY YYYYYYYY XXX XXXXXXXXXX + + makeFile $prefix AA V BB V CC V DD V + makeFile $prefix AAAAAAAAAA V BBBBBBBBBB V +done + +pushd $TMPDIR > /dev/null +getfattr -d * +ls -1i | sort +popd > /dev/null + +hardlink $TMPDIR --ignore-time --respect-xattrs --maximize + +pushd $TMPDIR > /dev/null +ls -1i | sort +popd > /dev/null + +rm -rf $TMPDIR diff --git a/test/test_patch.readme b/test/test_patch.readme new file mode 100644 index 0000000..81d17ff --- /dev/null +++ b/test/test_patch.readme @@ -0,0 +1,30 @@ +test_patch.sh [options] compare two folders, create and patch the first folder found and compare the result with the next folder found + -b | --base : base folder of versions + -c | --colorize : colorize output + -e | --encoding : (list of encoding algorithms) create patches with encoding algorithms given (could be 'raw'/'gzip'/'xz'/'zeros') + -h | --help : display this help and exit + -i | --ignore-identical: (y | n) compare file of old and new version before creating/applying/checking a patch (if file are same it's ignored, default is 'yes') + -k | --keep : keep template files and result + -o | --output : output trace. Verbosely list executed command (for terminal, use /dev/stderr instead of /dev/stdout=the standard command result) + -t | --tar : tar name of template files + -v | --valgrind : execute valgrind for each command + +test_patch takes as input a "base folder" (-b) or a tar file (-t). In any case, these must contain 2 others folders representing a list of files representing two different versions to compare, with the goal of : + - creating a patch for each file found + - applying these patches on the files of the old version + - checking if the patched files are effectively the same as the corresponding files in the new version +test_patch will list alphabeticaly the content of the "base folder", and take the first folder found as the "old version" and the next as the "new version" +and will create a 'result folder' which contains patches and patched files (stored in base folder with the folder name "'new version'_PATCH") + +test_patch will parse recursively the folder "old version", and create the corresponding tree in the result folder + +test_patch will create a patch for each file found if the file : +- is a file (!). The folders are not "patched" but parsed recursively. No analyse are done on folder. +- exists in the new version at the same place in the tree +- file size is not 0 byte in old or new version + +For each file, test_patch will encode the patch with the encoding algorithm given with the option (-e | --encoding). +This is a list, spaces separated, among the following encoding : 'raw'/'gzip'/'xz'/'zeros'/'any'. If this parameter is omitted, 'raw' is used. + +The simple command line to test is : +./test_patch.sh -t basedir_quick.tar -c diff --git a/test/test_patch.sh b/test/test_patch.sh new file mode 100755 index 0000000..f48c0d8 --- /dev/null +++ b/test/test_patch.sh @@ -0,0 +1,575 @@ +#!/bin/bash +shopt -s expand_aliases +templateFolder="./template" +LOG_MISS_TOOL="./miss_tool.txt" +unset trace_out +unset valgrind +unset template_tar_files +colorize=false +unset keep +errCount=0 +errLogFile="log_err.txt" +alias exec=exec_func +success=0 +error=1 +ignore=2 +alias return_on_error='err=$?; if [[ "$err" != "$success" ]]; then return $err; fi' +encodingList="raw" + +print_usage() { + echo $(basename $0)" [options] compare two folders, create and patch the first folder found and compare the result with the next folder found" + echo " The options are interpreted in the order in which they're entered!" + echo " -b | --base : base folder of versions" + echo " -c | --colorize : colorize output" + echo " -e | --encoding : (list of encoding algorithms) create patches with encoding algorithms given (could be 'raw'/'gzip'/'xz'/'zeros')" + echo " -h | --help : display this help and exit" + echo " -i | --ignore-identical: (y | n) compare file of old and new version before creating/applying/checking a patch (if file are same it's ignored, default is 'yes')" + echo " -k | --keep : keep template files and result" + echo " -o | --output : output trace. Verbosely list executed command (for terminal, use /dev/stderr instead of /dev/stdout=the standard command result)" + echo " -t | --tar : tar name of template files" + echo " -v | --valgrind : execute valgrind for each command" + exit 1 +} + +parse_options() { + options=$(getopt -s bash -o b:ce:hi:ko:t:v -l base:,colorize,encoding:,help,--ignore-identical:,keep,output:,tar:,valgrind -- "$@") + if [ "$?" != "0" ]; then + print_usage + exit + fi + eval set -- $options + while [ $# -gt 0 ] + do + case $1 in + -b|--base) + folderBase=$(echo $2 | sed "s/'//g") + shift + ;; + -c|--colorize) + colorize=true + ;; + -e|--encoding) + encodingList=$(to_lower "$2") + shift + ;; + -h|--help) + print_usage + ;; + -i|--ignore-identical) + compare=$(to_lower $2) + shift + ;; + -k|--keep) + keep=true + ;; + -o|--output) + trace_out=$(echo $2 | sed "s/'//g") + alias exec=exec_func_out + shift + ;; + -t|--tar) + template_tar_files=$(echo $2 | sed "s/'//g") + folderBase=$templateFolder + encodingList="raw gzip xz zeros any" + shift + ;; + -v|--valgrind) + valgrind="valgrind --trace-children=yes --track-fds=yes -q " + ;; + --) + shift; break + ;; + -*) + echo "$0: error - unrecognized option $1" 1>&2; exit 1 + ;; + (*) + break + ;; + esac + shift + done +} + +check_tool() { + local ERR=0 + CMD="which $@" + TOOL=$@ + HIDE_OUT=$(eval $CMD) + ERR=$? + if [ $ERR != "0" ]; then + echo $TOOL >> $LOG_MISS_TOOL + ERR=$error + fi + return $ERR +} + +check_dependencies() { + if [ -e $LOG_MISS_TOOL ]; then + rm $LOG_MISS_TOOL + fi + local ERR=0 + check_tool "bsdiff"; ERR=$(($ERR + $?)) + check_tool "bspatch"; ERR=$(($ERR + $?)) + check_tool "cmp"; ERR=$(($ERR + $?)) + + if [ $ERR != "0" ];then + log_err "check_tool" "$ERR" 'Missing command '$(cat $LOG_MISS_TOOL) + return $ERR + fi + + return $success +} + + +to_lower() { + echo $1 | tr '[:upper:]' '[:lower:]' +} + + +log_user() { + str=$@ + # do nothing it it's just a change of font attribute + if [ "${str:0:2}" = "\e" ] && [ ${#str} -le 5 ]; then + echo -en $str>/dev/stderr + return + fi + + if [ -z "$trace_out" ]; then + echo -e $str>/dev/stderr + else + trace $str + fi +} + + +log_debug() { + log_user $@ +} + +log_err() { + cmd=$1 + err=$2 + out=$3 + msg="\n$cmd return $fontRed$err$attrReset" + if [ "$out" != "" ]; then + msg="$msg and message \n$fontRed$out$attrReset\n" + fi + msg="$msg" + log_user -en $fontRed"\nERROR:$attrReset $msg" + echo -e "$msg" >>$errLogFile +} + +trace() { + str=$@ + if [ "$trace_out" != "" ]; then + if [ "${str:0:2}" = "\e" ] && [ ${#str} -le 5 ]; then + return + fi + echo -e $str>>$trace_out + fi +} + +exec_func_out() { + cmd=$@ + trace [ $fontCyan$cmd$attrReset ] + out=$(eval $cmd) + err=$? + if [ "$out" != "" ]; then # avoid empty echo + echo $out + fi + return $err +} + +exec_func() { + res=$(eval $@) + err=$? + if [ "$res" != "" ]; then + echo $res + fi + return $err +} + + +# Initialize main folder path. Get $1(=folder_BASE = path of the initial version) and determine : +# folderNextVers = path of the next version +# folderResult = path of the folder where the patch will be stored +set_folder_base() { + folderList=( $(exec "ls $1") ) + for folder in ${folderList[@]} + do + folder="$1/$folder" + if [ -d "$folder" ]; then + if [ -z $folderCurrVers ]; then + folderCurrVers=$folder + else + folderNextVers=$folder + break + fi + fi + done + folderResult="$folderNextVers""_PATCH" + relativeFolder="" + totalFiles=$(exec "ls -lR $folderCurrVers/") + totalFiles=$(exec "ls -lR $folderCurrVers/ | grep ^- | wc -l") + totalLink=$(exec "ls -lR $folderCurrVers/ | grep ^l | wc -l") + totalFileCount=$(($totalFiles + $totalLink )) +} + +# Increment statistic counter +inc_stat() { + local idx=$1 + local algo=$2 + + if [ "$algo" = "" ]; then + algo="filter" + fi + + eval stat=\$'stats_'$algo'_'$idx + eval stats'_'$algo'_'$idx=$(($stat + 1)) +} + + +# Before creating,applying,checking a patch, this function check than : +# this is effectively a file +# with a size in the old and previous version > 0 byte +# and extended attributes are the same +# because these cases are not managed by bsdiff +filter_file() { + local file_src=$1 + local file_cmp=$2 + local size_src + local size_cmp + local err=$ignore + + fileCount=$(($fileCount + 1)) + percent=$((($fileCount * 100) / $totalFileCount)) + if [ "$progress" != "$percent" ]; then + progress=$percent + log_user -n $fontCyan".$progress%"$attrReset + fi + + # not a file or not a link on file => ignore + [[ ! -f "$file_src" && ! -L "$file_src" ]] && return $ignore + + # not found in new version => ignore + [ ! -e "$file_cmp" ] && inc_stat idx_FileNotFound && inc_stat idx_FileIgnored && return $ignore + + # extended attributes has changed ? + xattrV1=( $(ls -Z $file_src) ) + xattrV2=( $(ls -Z $file_cmp) ) + [ "${xattrV1[0]}" != "${xattrV2[0]}" ] && inc_stat idx_xattrChanged && inc_stat idx_FileIgnored && return $ignore + + # File differe => analyze + size_src=$(exec "stat -c %s $file_src 2>/dev/null") + size_cmp=$(exec "stat -c %s $file_cmp 2>/dev/null") + [ "$size_src" != "$size_cmp" ] && inc_stat idx_FileAnalyzed && return $success + + # File size = 0 (in old and newest version) => ignore + [[ "$size_src" = "0" && "$size_cmp" = "0" ]] && inc_stat idx_FileEmpty && inc_stat idx_FileIgnored && return $ignore + + # Files unchanged => ignore + if [ "$compare" != "n" ]; then + CHANGE=$(eval "cmp -s $file_src $file_cmp") + [ "$?" = "0" ] && inc_stat idx_FileIgnored && inc_stat idx_FileUnchanged && return $ignore + fi + + # => analyze + inc_stat idx_FileAnalyzed + return $success + +} + +parse_folder() { + local folderSrc="$folderCurrVers/$relativeFolder" + local folderCmp="$folderNextVers/$relativeFolder" + local folderDest="$folderResult/$relativeFolder" + local err + local relativeFolderStore + + cmd="ls $folderSrc" + local file + fileList=$(exec "$cmd") + if [ "$?" != "0" ]; then + log_err "$cmd" "$?" "$fileList" + exit + fi + for file in $fileList + do + #if it is a folder (and not a link on folder), create the folder, and recurse into + if [[ -d "$folderSrc$file" && ! -L "$folderSrc$file" ]]; then + relativeFolderStore=$relativeFolder + relativeFolder="$relativeFolder$file/" + if [ ! -e "$folderDest$file" ]; then + exec "mkdir $folderDest$file" + fi + parse_folder + relativeFolder=$relativeFolderStore + err=2 + else + filter_file "$folderSrc$file" "$folderCmp$file" + err=$? + if [ "$err" = "$success" ]; then + test_patch $file + err=$? + fi + fi + done +} + +create_patch() { + local err + local file=$1 + local algo=$2 + local debug=$3 + fileSrc=$folderSrc$file + fileCmp=$folderCmp$file + filePatch="$folderDest$filename".$algo.patch + cmd="$valgrind $diffTool $fileSrc $fileCmp $filePatch $algo $debug" + out=$(exec "$cmd") + err=$? + if [ "$err" -lt "0" ]; then + log_err "$cmd" "$err" "$out" + inc_stat idx_ErrCreatePatch $algo + fi + return $err + +} + +apply_patch() { + local err + local file=$1 + local algo=$2 + oldFile=$folderSrc$file + newFile="$folderDest$filename".$algo.patched + patchFile="$folderDest$filename".$algo.patch + cmd="$valgrind $patchTool $oldFile $newFile $patchFile" + out=$(exec "$cmd 2>&1") + err=$? + if [ "$err" != "$success" ]; then + log_err "$cmd" "$err" "$out" + inc_stat idx_ErrApplyPatch $algo + fi + return $err +} + +check_result() { + local err + local file=$1 + local algo=$2 + patchedFile="$folderDest$filename".$algo.patched + newFile="$folderCmp$file" + cmd="$checkTool $patchedFile $newFile" + out=$(exec "$cmd 2>&1") + err=$? + if [ "$err" != "$success" ]; then + log_err "$cmd" "$err" "$out \n patch not well applied on '$folderDest$filename' witch patch '$folderDest$filename.$algo.patch'" + inc_stat idx_ErrCheckPatch $algo + fi + return $err +} + +test_patch() { + local file="$1" + filename=$(basename $file) + local err + local algo + if [ ! -z "$trace_out" ]; then + debug='debug' + fi + for algo in ${algoList[*]} + do + create_patch $file $algo $debug + [ "$?" = "0" ] && apply_patch $file $algo $debug + [ "$?" = "0" ] && check_result $file $algo $debug + done + return $? +} + + +def_font_attributes() { + fontCyan="\e[96m" + fontRed="\e[91m" + fontGreen="\e[92m" + fontBlue="\e[34m" + fontYellow="\e[93m" + fontWhite="\e[97m" + attrReset="\e[0m" +} + + +rm_logfile() { + if [[ "$trace_out" != "" && -e "$trace_out" && -f "$trace_out" ]]; then + exec "sudo rm $trace_out" + fi + if [ -e "$errLogFile" ]; then + exec "sudo rm $errLogFile" + fi +} + +rm_template() { + if [ "$keep" != "true" ]; then + if [[ "$template_tar_files" != "" && -d "$templateFolder" ]]; then + exec "sudo rm -rf $templateFolder" + fi + fi +} + +rm_result() { + if [ "$keep" != "true" ]; then + if [ -d "$folderResult" ]; then + exec "rm -rf $folderResult" + fi + fi +} + +test() { + parse_folder "$folderCurrVers" +} + +init() { + # Check than all variables are declared + # set -o nounset + compare='y' + parse_options "$@" + + diffTool=$(which bsdiff) + patchTool=$(which bspatch) + checkTool=$(which cmp) + checkTool="$checkTool -s " + + if [[ -z "$template_tar_files" && -z "$folderBase" ]]; then + print_usage + fi + if [ "$colorize" = "true" ]; then + def_font_attributes + fi + + check_dependencies + if [ "$?" != "$success" ]; then + exit $? + fi + rm_logfile + rm_template + + if [ "$encodingList" != "" ]; then + algoList=$encodingList + else + algoList="raw" + fi + + + if [ ! -e "$templateFolder" ]; then + exec "mkdir $templateFolder" + fi + if [ "$template_tar_files" != "" ]; then + exec "sudo tar --preserve-permissions --xattrs --selinux -p -xf $template_tar_files -C $templateFolder/" + fi + set_folder_base $folderBase + log_user "folder of current version (folderCurrVers)=$fontGreen$folderCurrVers$attrReset" + log_user "folder of next (newest) version (folderNextVers)=$fontGreen$folderNextVers$attrReset" + log_user "folder of patch (and patched) files (folderResult)=$fontGreen$folderResult$attrReset" + log_user "Found $fontGreen$totalFileCount$attrReset file/link" + log_user "diffTool used=$fontGreen$diffTool$attrReset" + log_user "patchTool used=$fontGreen$patchTool$attrReset" + log_user "checkTool used=$fontGreen$checkTool$attrReset" + + if [ -d "$folderResult" ]; then + read -e -i "Y" -p "A result folder have been found ($folderResult). Do you want to remove this folder? ('Y'es/'n'o/'q'uit): " + case $REPLY in + n | N) + # do nothing + ;; + y | Y) + exec "rm -rf $folderResult" + ;; + q | Q) + exit 0 + esac + fi + if [ ! -e "$folderResult" ]; then + exec "mkdir $folderResult" + fi + + stats=(0 0 0 0 0 0 0 0) + + idx_ErrCreatePatch=0 + idx_ErrApplyPatch=1 + idx_ErrCheckPatch=2 + idx_FileAnalyzed=3 + idx_FileIgnored=4 + idx_FileNotFound=5 + idx_FileUnchanged=6 + idx_FileEmpty=7 + idx_xattrChanged=8 +} + +show_stat() { + duration=$(($endTime - $startTime)) + log_user "\n-------------------------------------" + log_user "script start at $(date +%c --date @$startTime), ended at $(date +%c --date @$endTime)\n" + log_user "Time to process=$fontGreen$(date +%Mm:%Ss --date @$duration) \n$attrReset" + log_user "-------------------------------------" + [ "$stats_filter_idx_FileAnalyzed" = "" ] && stats_filter_idx_FileAnalyzed=0 + log_user $fontBlue"Analysed files : $fontGreen $stats_filter_idx_FileAnalyzed"$attrReset + + [ "$stats_filter_idx_FileIgnored" = "" ] && stats_filter_idx_FileIgnored=0 + log_user $fontBlue"Ignored files : $stats_filter_idx_FileIgnored"$attrReset + + [ "$stats_filter_idx_FileEmpty" = "" ] && stats_filter_idx_FileEmpty=0 + log_user $fontBlue".Empty files : $stats_filter_idx_FileEmpty"$attrReset + [ "$stats_filter_idx_FileNotFound" = "" ] && stats_filter_idx_FileNotFound=0 + log_user $fontBlue".Not Found in new version : $stats_filter_idx_FileNotFound"$attrReset + [ "$stats_filter_idx_xattrChanged" = "" ] && stats_filter_idx_xattrChanged=0 + log_user $fontBlue".Extended attributes have changed : $stats_filter_idx_xattrChanged"$attrReset + if [ "$compare" != "n" ]; then + [ "$stats_filter_idx_FileUnchanged" = "" ] && stats_filter_idx_FileUnchanged=0 + log_user $fontBlue".Unchanged files : $stats_filter_idx_FileUnchanged"$attrReset + fi + errCount=0 + for algo in ${algoList[*]} + do + eval stat_CreatePatch=\$'stats_'$algo'_idx_ErrCreatePatch'; [ "$stat_CreatePatch" = "" ] && stat_CreatePatch=0 + eval stat_ApplyPatch=\$'stats_'$algo'_idx_ErrApplyPatch'; [ "$stat_ApplyPatch" = "" ] && stat_ApplyPatch=0 + eval stat_CheckPatch=\$'stats_'$algo'_idx_ErrCheckPatch'; [ "$stat_CheckPatch" = "" ] && stat_CheckPatch=0 + + eval errCount=$(( $stat_CreatePatch + $stat_ApplyPatch + $stat_CheckPatch )) + if [ "$errCount" = 0 ] ; then + log_user $fontGreen"With '$algo' encoding, count $errCount errors." + else + log_user $fontRed"With '$algo' encoding, count $errCount errors, which is :" + + eval stat=\$'stats_'$algo'_idx_ErrCreatePatch'; [ "$stat" = "" ] && stat=0 + log_user $fontBlue".on patch created : $fontRed $stat"$attrReset + + eval stat=\$'stats_'$algo'_idx_ErrApplyPatch'; [ "$stat" = "" ] && stat=0 + log_user $fontBlue".on patch applied : $fontRed $stat"$attrReset + + eval stat=\$'stats_'$algo'_idx_ErrCheckPatch'; [ "$stat" = "" ] && stat=0 + log_user $fontBlue".on patch checked : $fontRed $stat"$attrReset + fi + done + if [ "$errCount" -gt "0" ]; then + log_user $fontRed"see '$errLogFile' for more informations" + log_user $attrReset + fi +} + +cleanup() { + rm_template + rm_result +} + +main() { + PATH_STORE=$PATH + PATH=.:..:../..:$PATH + startTime=$(date +%s) + init "$@" + parse_folder "$folderCurrVers" + endTime=$(date +%s) + show_stat + cleanup + PATH=$PATH_STORE +} + +main "$@" +