clearlinux/koji-setup-scripts
2
0
Fork 0
mirror of https://github.com/clearlinux/koji-setup-scripts.git synced 2026-04-28 11:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace mash utility with bash script

Browse Source 
The mash utility has not been ported to python3.  There is also no
comparable replacement utility of a similar scope that can do a similar
job.  There is pungi utility, which is the replacement for mash.
However, pungi is Fedora's build process encapsulated into a single tool
and is doing too much for simply creating repos for distribution.

To avoid this complexity, koji does have a dist-repo subcommand.
However, this takes too long to execute at around 15 minutes on powerful
hardware.

In lieu of not wanting to maintain mash any further, write a small bash
script that performs the same function using artifacts for koji on the
disk.

Signed-off-by: George T Kramer <george.t.kramer@intel.com>
This commit is contained in:
George T Kramer
2019-09-18 16:01:33 -07:00
committed by Patrick McCarty
parent ed3f44ba99
commit 8e0085e302
1 changed files with 78 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
koji-setup/mash.sh
View File

@@ -3,11 +3,8 @@
# SPDX-License-Identifier: Apache-2.0
set -e
if [[ -e /etc/profile.d/proxy.sh ]]; then
source /etc/profile.d/proxy.sh
fi
. /etc/profile.d/proxy.sh || :
TAG_NAME="${TAG_NAME:-clear}"
BUILD_ARCH="${BUILD_ARCH:-x86_64}"
KOJI_DIR="${KOJI_DIR:-/srv/koji}"
MASH_DIR="${MASH_DIR:-/srv/mash}"
@@ -16,10 +13,80 @@ MASH_TRACKER_DIR="$MASH_DIR"/latest
MASH_DIR_OLD="$MASH_TRACKER_DIR".old
MASH_DIR_NEW="$MASH_TRACKER_DIR".new
create_dist_repos() {
local output_dir="${1}"
local work_dir="$(mktemp -d)"
local nvr_pkg_list="${work_dir}/nvr-pkg-list"
local bin_rpm_paths="${work_dir}/bin-rpm-paths"
local debuginfo_rpm_paths="${work_dir}/debuginfo-rpm-paths"
local src_rpm_paths="${work_dir}/src-rpm-paths"
local comps_file="${work_dir}/comps.xml"
sed -r -e 's|[^/]+/||' -e "s|^|${KOJI_DIR}/|" "${KOJI_REPO_PATH}/${BUILD_ARCH}/pkglist" > "${bin_rpm_paths}"
cut -d/ -f3-5 "${KOJI_REPO_PATH}/${BUILD_ARCH}/pkglist" | sort -u > "${nvr_pkg_list}"
while IFS='/' read -r name version release; do
local debuginfo_rpm_path="${KOJI_DIR}/packages/${name}/${version}/${release}/${BUILD_ARCH}/${name}-debuginfo-${version}-${release}.${BUILD_ARCH}.rpm"
if [[ -s "${debuginfo_rpm_path}" ]]; then
echo "${debuginfo_rpm_path}" >> "${debuginfo_rpm_paths}"
fi
echo "${KOJI_DIR}/packages/${name}/${version}/${release}/src/${name}-${version}-${release}.src.rpm" >> "${src_rpm_paths}"
done < "${nvr_pkg_list}"
cp -f "${KOJI_REPO_PATH}/groups/comps.xml" "${comps_file}"
make_repo "${output_dir}/clear/${BUILD_ARCH}/os" "Packages" "${bin_rpm_paths}" "${comps_file}" &
make_repo "${output_dir}/clear/${BUILD_ARCH}/debug" "." "${debuginfo_rpm_paths}" &
make_repo "${output_dir}/clear/source/SRPMS" "." "${src_rpm_paths}" &
wait
create_dnf_conf "${work_dir}/dnf-os.conf" "${output_dir}/clear/${BUILD_ARCH}/os" clear-os
create_dnf_conf "${work_dir}/dnf-debug.conf" "${output_dir}/clear/${BUILD_ARCH}/debug" clear-debug
create_dnf_conf "${work_dir}/dnf-SRPMS.conf" "${output_dir}/clear/source/SRPMS" clear-SRPMS
write_packages_file "${work_dir}/dnf-os.conf" "$output_dir/clear/$BUILD_ARCH/packages-os"
write_packages_file "${work_dir}/dnf-debug.conf" "$output_dir/clear/$BUILD_ARCH/packages-debug"
write_packages_file "${work_dir}/dnf-SRPMS.conf" "$output_dir/clear/source/packages-SRPMS"
rm -rf "${work_dir}"
}
make_repo() {
local repo_dir="${1}"
local rpm_dir="${1}/${2}"
local file_list="${3}"
local comps_file="${4}"
local create_repo_cmd="createrepo_c --quiet --database --compress-type xz --workers $(nproc --all)"
mkdir -p "${rpm_dir}"
xargs -a "${file_list}" -I {} cp -f {} "${rpm_dir}"
if [[ -z "${comps_file}" ]]; then
${create_repo_cmd} "${repo_dir}"
else
${create_repo_cmd} --groupfile "${comps_file}" "${repo_dir}"
fi
}
create_dnf_conf() {
local dnf_conf="${1}"
local repo_path="${2}"
local repo_name="${3:-clear}"
cat > "${dnf_conf}" <<EOF
[${repo_name}]
name=${repo_name}
baseurl=file://${repo_path}
EOF
}
write_packages_file() {
local PKG_DIR="$1"
local PKG_FILE="$2"
rpm -qp --qf="%{NAME}\t%{VERSION}\t%{RELEASE}\n" "$PKG_DIR"/*.rpm | sort > "$PKG_FILE"
local dnf_conf="${1}"
local output_file="${2}"
dnf --config "${dnf_conf}" --quiet --releasever=clear \
repoquery --all --queryformat="%{NAME}\t%{VERSION}\t%{RELEASE}" \
| sort > "${output_file}"
}
if [[ -e "$MASH_TRACKER_FILE" ]]; then
@@ -27,19 +94,13 @@ if [[ -e "$MASH_TRACKER_FILE" ]]; then
else
MASH_BUILD_NUM=0
fi
KOJI_BUILD_NUM="$(basename "$(realpath "$KOJI_DIR"/repos/dist-"$TAG_NAME"-build/latest/)")"
KOJI_TAG="${KOJI_TAG:-"dist-clear"}"
KOJI_REPO_PATH="$(realpath "$KOJI_DIR/repos/$KOJI_TAG-build/latest")"
KOJI_BUILD_NUM="$(basename "$KOJI_REPO_PATH")"
if [[ "$MASH_BUILD_NUM" -ne "$KOJI_BUILD_NUM" ]]; then
COMPS_FILE="$(mktemp)"
koji show-groups --comps dist-"$TAG_NAME"-build > "$COMPS_FILE"
rm -rf "$MASH_DIR_NEW"
mkdir -p "$MASH_DIR_NEW"
mash --outputdir="$MASH_DIR_NEW" --compsfile="$COMPS_FILE" clear
rm -f "$COMPS_FILE"
write_packages_file "$MASH_DIR_NEW"/clear/"$BUILD_ARCH"/os/Packages "$MASH_DIR_NEW"/clear/"$BUILD_ARCH"/packages-os
write_packages_file "$MASH_DIR_NEW"/clear/"$BUILD_ARCH"/debug "$MASH_DIR_NEW"/clear/"$BUILD_ARCH"/packages-debug
write_packages_file "$MASH_DIR_NEW"/clear/source/SRPMS "$MASH_DIR_NEW"/clear/source/packages-SRPMS
create_dist_repos "$MASH_DIR_NEW"
if [[ -e "$MASH_TRACKER_DIR" ]]; then
mv "$MASH_TRACKER_DIR" "$MASH_DIR_OLD"
fi