bundle-chroot-builder/bundle-chroot-builder.py

#!/usr/bin/env python3
#
# Copyright © 2015-2016 Intel Corporation.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 or later of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Software Update server bundle chroot creator"""

# pylint: disable=R0912
# pylint: disable=R0914
# pylint: disable=R0915
# pylint: disable=W0703

import argparse
import configparser
import os
import os.path
import platform
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import urllib.request
import multiprocessing


def handle_options():
    """Process command arguments"""
    aparser = argparse.ArgumentParser()
    aparser.add_argument("version", help="New build version")
    aparser.add_argument("-m", "--mix-version", help="Mix version for SWUPD mixer")
    aparser.add_argument("-c", "--config", help="Path to config file")
    args = aparser.parse_args()
    return args


def read_config(args):
    buildconf='/usr/share/defaults/bundle-chroot-builder/builder.conf'
    if os.path.isfile('/etc/bundle-chroot-builder/builder.conf'):
        buildconf = '/etc/bundle-chroot-builder/builder.conf'
    if args.config:
        buildconf = args.config
    config = configparser.ConfigParser()
    config.read(buildconf)

    """Read the configuration file for our script values"""
    conf = config['Builder']
    state_dir = conf['SERVER_STATE_DIR']
    bundles = conf['BUNDLE_DIR']
    yum_conf = conf['YUM_CONF']

    return state_dir, bundles, yum_conf


def install_bundle(out_dir, postfix, bundle, bundles, yum_cmd):
    """Helper function to yum install a bundle"""
    m4 = subprocess.Popen(["m4", bundles + "/" + bundle], cwd=bundles, stdout=subprocess.PIPE)
    m4.wait()
    pkgs = m4.stdout.readlines()
    to_install = []
    for pkg in pkgs:
        pkg = pkg.decode("utf-8").strip()
        # Don't add blank lines or lines with leading '#'
        if len(pkg) == 0 or pkg[0] == "#":
            continue
        to_install.append(pkg)
    subprocess.check_output(yum_cmd + ["--installroot={0}/{1}" .format(out_dir, postfix), "install"] + to_install)
    includes = []
    with open(bundles + "/" + bundle, "r") as bfile:
        for line in bfile.readlines():
            if line.startswith("include("):
                # 8 characters skips 'include('
                end_position = line.find(")")
                includes.append(line[8:end_position] + "\n")
    with open("{0}/{1}-includes".format(out_dir, bundle), "w") as ifile:
        ifile.writelines(includes)


def process_bundle(out_dir, bundle, bundles, yum_cmd):
    subprocess.check_output(["cp", "-a", "--preserve=all", "{}/os-core".format(out_dir), "{0}/{1}".format(out_dir, bundle)])
    install_bundle(out_dir, bundle, bundle, bundles, yum_cmd)
    with open(out_dir + "/packages-{}".format(bundle), "w") as file:
        subprocess.Popen(['rpm', '--root={0}/{1}'.format(out_dir, bundle),
                          '-qa', '--queryformat', '%{NAME}\t%{SOURCERPM}\n'], stdout=file).wait()
    with open(out_dir + "/packages-{}".format(bundle), "r") as file:
        raw_list = file.readlines()
    for line in raw_list:
        # Line is 'pkg\tpkg.src.rpm\n'
        subpackage, srpm = line.split("\t")
        # Drop newline
        srpm = srpm[:-1]
        with open(out_dir + "/pkgmap-{0} * {1}".format(bundle, srpm), "a") as file:
            subprocess.Popen(['rpm', '--root={0}/{1}'.format(out_dir, bundle),
                              '-ql', subpackage], stdout=file).wait()
    bundle_prefix = out_dir + "/" + bundle + "/usr/share/clear/bundles/"
    with open(bundle_prefix + bundle, "a"):
        os.utime(bundle_prefix + bundle, None)
    clean_bundle(out_dir, bundle, bundles, yum_cmd)


def clean_bundle(out_dir, bundle, bundles, yum_cmd):
    mode_00755 = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    mode_01777 = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_ISVTX
    prev_dir = os.getcwd()
    os.chdir(out_dir + "/" + bundle)
    # must use relative paths here or we kill the host
    shutil.rmtree("./var/lib/")
    os.mkdir("./var/lib/")
    os.chmod("./var/lib/", mode_00755)
    shutil.rmtree("./var/cache/")
    os.mkdir("./var/cache/")
    os.chmod("./var/cache/", mode_00755)
    shutil.rmtree("./var/log/")
    os.mkdir("./var/log/")
    os.chmod("./var/log/", mode_00755)
    shutil.rmtree("./dev/")
    os.mkdir("./dev/")
    os.chmod("./dev/", mode_00755)
    shutil.rmtree("./run/")
    os.mkdir("./run/")
    os.chmod("./run/", mode_00755)
    shutil.rmtree("./tmp/")
    os.mkdir("./tmp/")
    os.chmod("./tmp/", mode_01777)
    os.chdir(prev_dir)


def create_chroots(args, state_dir, bundles, yum_conf):
    """The state_dir should always be created if it does not exist"""
    if os.path.isdir(state_dir) == False:
        os.makedirs(state_dir)
    if os.path.isdir(state_dir + "/image") == False:
        os.makedirs(state_dir + "/image")
    if os.path.isdir(state_dir + "/www") == False:
        os.makedirs(state_dir + "/www")
    if os.path.isdir(state_dir + "/image/0") == False:
        os.makedirs(state_dir + "/image/0")
    if os.path.isdir(state_dir + "/www/0") == False:
        os.makedirs(state_dir + "/www/0")

    """Create server.ini and groups.ini for create_update later on"""
    with open(state_dir + "/server.ini", "w+") as serverini:
        serverini.write("[Server]\nemptydir=" + state_dir + "/empty/\nimagebase=" + state_dir + "/image/\noutputdir=" + state_dir + "/www/\n")
    with open(state_dir + "/groups.ini", "w+") as groupsini:
        for bundle in os.listdir(bundles):
            groupsini.write("[" + bundle + "]\ngroup=" + bundle + "\n\n")

    """Setup chroots for bundles"""
    bversion = ""
    blines = []
    if args.mix_version:
        out_version = args.mix_version
    else:
        out_version = args.version
    build_version = args.version

    if os.path.exists(state_dir + "/image/latest.version") == False:
        with open(state_dir + "/image/latest.version", "w") as latestver:
            latestver.write("0\n")

    config = configparser.ConfigParser()
    config.read(yum_conf)

    """Read the yum config to find which url to use"""
    if 'local' in config:
        conf_baseurl = config['local']['baseurl']
        print(conf_baseurl)
    else:
        conf_baseurl = config['clear']['baseurl']
        url = conf_baseurl.split("$releasever")[0] + build_version + conf_baseurl.split("$releasever")[1]
        conf_baseurl = url

    out_dir = state_dir + "/image/" + out_version

    if platform.dist()[0] == "fedora" and int(platform.dist()[1]) >= 22:
        print("using dnf instead of yum")
        packager = ["dnf"]
    else:
        packager = ["yum"]
    yum_cmd = packager + ["--config={}".format(yum_conf), "-y", "--noplugins", "--releasever={}".format(build_version)]
    if 'local' in config == False:
        try:
            urllib.request.urlopen(conf_baseurl)
        except Exception as excep:
            print("Unable to retrieve {}".format(conf_baseurl))
            print(excep)
            sys.exit(-2)
    if os.path.isdir(out_dir):
        print("Removing pre-existing {} before starting".format(out_dir))
        os.system('rm -rf '+out_dir)
    # if os.path.isdir(out_dir):
    #     print("Removing pre-existing {} before starting".format(out_dir))
    #     shutil.rmtree(out_dir)
    print("Preparing new {}".format(out_dir))
    print("  based on bundles from: {}".format(bundles))
    print("  and yum config: {}".format(yum_conf))

    print("Creating os-core bundle")
    os.makedirs(out_dir + "/os-core/var/lib/rpm")

    print("Initializing rpm database")
    subprocess.check_output(["rpm", "--root", "{}/os-core".format(out_dir), "--initdb"])

    print("Cleaning yum cache")
    subprocess.check_output(yum_cmd + ["--installroot={}/os-core".format(out_dir), "clean", "all"])

    print("Yum installing os-core filesystem")
    subprocess.check_output(yum_cmd + ["--installroot={}/os-core".format(out_dir), "install", "filesystem"])

    print("Yum installing packages from os-core")
    install_bundle(out_dir, "os-core", "os-core", bundles, yum_cmd)

    os.makedirs(out_dir + "/os-core/usr/share/clear", exist_ok=True)
    with open(out_dir + "/os-core/usr/share/clear/version", "w") as file:
        file.writelines([out_version])
    with open(out_dir + "/os-core/usr/share/clear/versionstamp", "w") as file:
        file.writelines([str(int(time.time()))])

    bundle_prefix = out_dir + "/os-core/usr/share/clear/bundles/"
    os.makedirs(bundle_prefix)
    with open(bundle_prefix + "os-core", "a"):
        os.utime(bundle_prefix + "os-core", None)

    print("Noting os-core package list")
    with open(out_dir + "/versions", "w") as file:
        subprocess.Popen(yum_cmd + ["--installroot={}/os-core".format(out_dir), "list"], stdout=file).wait()
    with open(out_dir + "/packages-os-core", "w") as file:
        subprocess.Popen(['rpm', '--root={}/os-core'.format(out_dir),
                          '-qa', '--queryformat', '%{NAME}\t%{SOURCERPM}\n'], stdout=file).wait()
    with open(out_dir + "/os-core/usr/lib/os-release", "r") as file:
        lines = file.readlines()
        for line in lines:
            if line.startswith("VERSION_ID="):
                bversion = line.split("=")[1]
                if bversion != out_version:
                    line = "VERSION_ID={}\n".format(out_version)
            blines.append(line)
    with open(out_dir + "/os-core/usr/lib/os-release", "w") as file:
        file.writelines(blines)

    bundle_list = os.listdir(bundles)
    bundle_list.remove('os-core')
    pool = multiprocessing.Pool()
    results_list = []

    def progress(r):
        print('.', end="", flush=True)

    for bundle in bundle_list:
        print("Scheduling to process bundle {}".format(bundle))
        r = pool.apply_async(process_bundle, (out_dir, bundle, bundles, yum_cmd), callback=progress)
        results_list.append(r)
    print("Waiting for tasks to complete...")
    pool.close()
    pool.join()
    print(' done!')
    print("Retrieving results...")
    for r in results_list:
        r.get()

    """Read the URL values from builder.conf and insert them into os-core-update to swupd knows where to pull content from"""
    buildconf='/usr/share/defaults/bundle-chroot-builder/builder.conf'
    if os.path.isfile('/etc/bundle-chroot-builder/builder.conf'):
        buildconf = '/etc/bundle-chroot-builder/builder.conf'
    if args.config:
        buildconf = args.config
    config = configparser.ConfigParser()
    config.read(buildconf)

    """Read the configuration file for our script values"""
    conf = config['swupd']
    bundlename = conf['BUNDLE']
    url = conf['URL']
    contenturl = conf['CONTENTURL']
    versionurl = conf['VERSIONURL']
    formatname = conf['FORMAT']

    confpath = os.path.join(out_dir, bundlename, "/usr/share/defaults/swupd/")
    os.mkdir(confpath)
    with open(os.path.join(confpath, "url"), "w") as file:
        file.writelines(url)
    with open(os.path.join(confpath, "contenturl"), "w") as file:
        file.writelines(contenturl)
    with open(os.path.join(confpath, "versionurl"), "w") as file:
        file.writelines(versionurl)
    with open(os.path.join(confpath, "format"), "w") as file:
        file.writelines(formatname)

    print("Creating package to file mappings")
    package_mapping = {}
    map_files = [f for f in os.listdir(out_dir) if f.startswith("pkgmap-")]
    for map_file in map_files:
        # map_file's name is "pkgmap-{bundle} * {pkg}.src.rpm"
        # The " * " is pure magic but unlikely enough to not show up in a
        # real package's filename I'm using it instead of recursing through
        # temporary per bundle folders
        package_name = map_file[map_file.find(" * ") + 3:-len(".src.rpm")]
        with open(out_dir + "/" +  map_file, "r") as file:
            path_list = file.readlines()
        if package_name not in package_mapping:
            package_mapping[package_name] = set()
        for path in path_list:
            package_mapping[package_name].add(path)
    for package_name, paths in package_mapping.items():
        with open(out_dir + "/files-{}".format(package_name), "w") as file:
            file.writelines(sorted(paths))
    for map_file in map_files:
        os.unlink(out_dir + "/" + map_file)

    print("Cleaning os-core...")
    clean_bundle(out_dir, 'os-core', bundles, yum_cmd)

    print("Cleaning package list")
    web_dir = state_dir + "/www/" + out_version + "/"
    if os.path.isdir(web_dir):
        print("  removing pre-existing {} before setting version file" .format(web_dir))
        os.system('rm -rf '+web_dir)
    # FIXME: Figure out problems with the below to replace the above.
    # Consider all uses of os.system("rm -fr") as FIXMEs of the same variety.
    # if os.path.isdir(web_dir):
    #     print("  removing pre-existing {} before setting version file"
    #           .format(web_dir))
    #     shutil.rmtree(web_dir)
    os.makedirs(web_dir + '/noship')
    versions_output = []
    with open(out_dir + "/versions", "r") as file:
        versions = set()
        lines = file.readlines()
        for line in lines:
            if line.startswith("Available") or \
               line.startswith("Installed") or \
               line.startswith("BDB2053") or \
               line.startswith("rpm") or line.startswith("yum"):
                continue
            name, pver = re.search("^([^\t ]*)[\t ]*([^\t ]*)",
                                   line).groups()
            versions.add(name + ":" + pver)
        versions = sorted(versions)
        versions_output.append("{0: <50}{1}\n".format("Available", "Packages"))
        for name, pver in [line.split(":") for line in versions]:
            versions_output.append("{0: <50}{1}\n".format(name, pver))
    with open(web_dir + "versions", "w") as file:
        file.writelines(versions_output)
    for bundle in os.listdir(bundles):
        shutil.copyfile(out_dir + "/packages-{}".format(bundle), web_dir + "/noship/packages-{}".format(bundle))
        if os.path.isfile(out_dir + "/{}-includes".format(bundle)):
            shutil.copyfile(out_dir + "/{}-includes".format(bundle), web_dir + "/noship/{}-includes".format(bundle))
    for package_name in package_mapping.keys():
        shutil.copyfile(out_dir + "/files-{}".format(package_name),
                        web_dir + "/noship/files-{}".format(package_name))


def main():
    """Entry point for chroot creator"""
    args = handle_options()
    state_dir, bundles, yum_conf = read_config(args)
    create_chroots(args, state_dir, bundles, yum_conf)

if __name__ == '__main__':
    main()
    sys.exit(0)