Deploy all DevOps services for development

A deployment of a "koji" server consists of additional services that support development beyond koji itself. These include git/cgit/gitolite, a mash containing the current snapshot of the packages in koji, and an upstreams package sources cache. Because these are required for integration with a DevOps workflow, they are not included by default in the ansible playbook.
2026-04-28 11:03:50 +00:00 · 2018-10-22 15:00:19 -07:00
parent 17ac27a687
commit 9fcbf230a4
12 changed files with 566 additions and 155 deletions
--- a/README.md
+++ b/README.md
@@ -0,0 +1,2 @@
+Assumes:
+- All scripts are run as the root user
--- a/koji-setup/bootstrap-build.sh
+++ b/koji-setup/bootstrap-build.sh
@@ -1,18 +1,31 @@
 #!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
 set -xe
 SCRIPT_DIR="$(dirname "$(realpath "$0")")"
 source "$SCRIPT_DIR"/parameters.sh

+if [[ -n "$SRC_RPM_DIR" && -n "$BIN_RPM_DIR" ]]; then
+	sudo -u kojiadmin koji import "$SRC_RPM_DIR"/*.src.rpm
+	sudo -u kojiadmin koji import "$BIN_RPM_DIR"/*."$RPM_ARCH".rpm
+	if [[ -n "$DEBUG_RPM_DIR" ]]; then
+		sudo -u kojiadmin koji import "$DEBUG_RPM_DIR"/*."$RPM_ARCH".rpm
+	fi
+fi
 sudo -u kojiadmin koji add-tag dist-"$TAG_NAME"
-sudo -u kojiadmin koji add-tag --parent dist-"$TAG_NAME" --arches "x86_64" dist-"$TAG_NAME"-build
 sudo -u kojiadmin koji edit-tag dist-"$TAG_NAME" -x mock.package_manager=dnf
+if [[ -n "$SRC_RPM_DIR" && -n "$BIN_RPM_DIR" ]]; then
+	sudo -u kojiadmin koji list-pkgs --quiet | xargs -I {} sudo -u kojiadmin koji add-pkg --owner kojiadmin dist-"$TAG_NAME" {}
+	sudo -u kojiadmin koji list-untagged | xargs -n 1 -I {} sudo -u kojiadmin koji call tagBuildBypass dist-"$TAG_NAME" {}
+fi
+sudo -u kojiadmin koji add-tag --parent dist-"$TAG_NAME" --arches "$RPM_ARCH" dist-"$TAG_NAME"-build
+sudo -u kojiadmin koji add-target dist-"$TAG_NAME" dist-"$TAG_NAME"-build
 sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build build
 sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build srpm-build
-sudo -u kojiadmin koji add-target dist-"$TAG_NAME" dist-"$TAG_NAME"-build
+sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build build autoconf automake automake-dev binutils bzip2 clr-rpm-config coreutils diffutils g++ gawk gcc gcc-dev gettext gettext-bin git glibc-dev glibc-locale glibc-utils grep gzip hostname lib6-locale libc6-dev libcap libgcc-s-dev libstdc++-dev libtool libtool-dev linux-libc-headers linux-libc-headers-dev m4 make netbase nss-altfiles patch pigz pkg-config pkg-config-dev rpm-build sed sed-doc shadow systemd-libs tar unzip which xz
+sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build srpm-build coreutils cpio curl-bin git glibc-utils grep gzip make pigz rpm-build sed shadow tar unzip wget xz
 if [[ -n "$EXTERNAL_REPO" ]]; then
 	sudo -u kojiadmin koji add-external-repo -t dist-"$TAG_NAME"-build dist-"$TAG_NAME"-external-repo "$EXTERNAL_REPO"
 fi
-sudo -u kojiadmin koji taginfo dist-"$TAG_NAME"-build
-sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build build autoconf automake automake-dev binutils bzip2 clr-rpm-config coreutils diffutils g++ gawk gcc gcc-dev gettext gettext-bin git glibc-dev glibc-locale glibc-utils grep gzip hostname lib6-locale libc6-dev libcap libgcc-s-dev libstdc++-dev libtool libtool-dev linux-libc-headers linux-libc-headers-dev m4 make netbase nss-altfiles patch pigz pkg-config pkg-config-dev rpm-build sed sed-doc shadow systemd-libs tar unzip which xz
-sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build srpm-build coreutils cpio curl-bin git glibc-utils grep gzip make pigz rpm-build sed shadow tar unzip wget xz
 sudo -u kojiadmin koji regen-repo dist-"$TAG_NAME"-build
--- a/koji-setup/deploy-git.sh
+++ b/koji-setup/deploy-git.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/parameters.sh
+
+swupd bundle-add scm-server httpd
+
+## GITOLITE SETUP
+mkdir -p "$GIT_DIR"
+chown -R "$GIT_USER":"$GIT_USER" "$GIT_DIR"
+# Add symlink for backwards compatibility
+if [[ "$GIT_DIR" != "$GIT_DEFAULT_DIR" ]]; then
+	if [ "$(ls -A "$GIT_DEFAULT_DIR")" ]; then
+		mv "$GIT_DEFAULT_DIR" "$GIT_DEFAULT_DIR".old
+	else
+		rm -rf "$GIT_DEFAULT_DIR"
+	fi
+	ln -sf "$GIT_DIR" "$GIT_DEFAULT_DIR"
+	chown -h "$GIT_USER":"$GIT_USER" "$GIT_DEFAULT_DIR"
+fi
+GITOLITE_PUB_KEY_FILE="$GIT_DEFAULT_DIR/gitolite.pub"
+echo "$GITOLITE_PUB_KEY" > "$GITOLITE_PUB_KEY_FILE"
+chown "$GIT_USER":"$GIT_USER" "$GITOLITE_PUB_KEY_FILE"
+sudo -u "$GIT_USER" gitolite setup -pk "$GITOLITE_PUB_KEY_FILE"
+
+
+if $IS_ANONYMOUS_GIT_NEEDED; then
+	## GIT PROTOCOL CLONING
+	mkdir -p /etc/systemd/system
+	cat > /etc/systemd/system/git-daemon.service <<- EOF
+	[Unit]
+	Description=Git Daemon
+
+	[Service]
+	ExecStart=/usr/bin/git daemon --reuseaddr --base-path=$GIT_DEFAULT_DIR $GIT_DEFAULT_DIR
+
+	Restart=always
+	RestartSec=500ms
+
+	User=$GIT_USER
+	Group=$GIT_USER
+
+	[Install]
+	WantedBy=multi-user.target
+	EOF
+	systemctl daemon-reload
+	systemctl enable --now git-daemon
+
+
+	## CGIT WEB INTERFACE
+	cat > /etc/cgitrc <<- EOF
+	# Enable caching of up to 1000 output entries
+	cache-size=10
+
+	# Specify the css url
+	css=/cgit-data/cgit.css
+
+	# Show extra links for each repository on the index page
+	enable-index-links=1
+
+	# Enable ASCII art commit history graph on the log pages
+	enable-commit-graph=1
+
+	# Show number of affected files per commit on the log pages
+	enable-log-filecount=1
+
+	# Show number of added/removed lines per commit on the log pages
+	enable-log-linecount=1
+
+	# Use a custom logo
+	logo=/cgit-data/cgit.png
+
+	# Enable statistics per week, month and quarter
+	max-stats=quarter
+
+	# Allow download of tar.gz, tar.bz2 and zip-files
+	snapshots=tar.gz tar.bz2
+
+	##
+	## List of common mimetypes
+	##
+	mimetype.gif=image/gif
+	mimetype.html=text/html
+	mimetype.jpg=image/jpeg
+	mimetype.jpeg=image/jpeg
+	mimetype.pdf=application/pdf
+	mimetype.png=image/png
+	mimetype.svg=image/svg+xml
+
+	# Enable syntax highlighting and about formatting
+	source-filter=/usr/libexec/cgit/filters/syntax-highlighting.py
+	about-filter=/usr/libexec/cgit/filters/about-formatting.sh
+
+	##
+	## List of common readmes
+	##
+	readme=:README.md
+	readme=:readme.md
+	readme=:README.mkd
+	readme=:readme.mkd
+	readme=:README.rst
+	readme=:readme.rst
+	readme=:README.html
+	readme=:readme.html
+	readme=:README.htm
+	readme=:readme.htm
+	readme=:README.txt
+	readme=:readme.txt
+	readme=:README
+	readme=:readme
+	readme=:INSTALL.md
+	readme=:install.md
+	readme=:INSTALL.mkd
+	readme=:install.mkd
+	readme=:INSTALL.rst
+	readme=:install.rst
+	readme=:INSTALL.html
+	readme=:install.html
+	readme=:INSTALL.htm
+	readme=:install.htm
+	readme=:INSTALL.txt
+	readme=:install.txt
+	readme=:INSTALL
+	readme=:install
+
+	# Direct cgit to repository location managed by gitolite
+	remove-suffix=1
+	project-list=$GIT_DEFAULT_DIR/projects.list
+	scan-path=$GIT_DEFAULT_DIR/repositories
+	EOF
+
+	mkdir -p /etc/httpd/conf.modules.d
+	cat > /etc/httpd/conf.modules.d/cgid.conf <<- EOF
+	LoadModule cgid_module lib/httpd/modules/mod_cgid.so
+	ScriptSock /run/httpd/cgid.sock
+	EOF
+
+	mkdir -p /etc/httpd/conf.d
+	cat > /etc/httpd/conf.d/cgit.conf <<- EOF
+	Alias /cgit-data /usr/share/cgit
+	<Directory "/usr/share/cgit">
+	    AllowOverride None
+	    Options None
+	    Require all granted
+	</Directory>
+
+	ScriptAlias /cgit /usr/libexec/cgit/cgi-bin/cgit
+	<Directory "/usr/libexec/cgit">
+	    AllowOverride None
+	    Options ExecCGI
+	    Require all granted
+	</Directory>
+	EOF
+	usermod -a -G "$GIT_USER" "$HTTPD_USER"
+
+	systemctl restart httpd
+	systemctl enable httpd
+fi
--- a/koji-setup/deploy-koji-builder.sh
+++ b/koji-setup/deploy-koji-builder.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/parameters.sh
+
+# Install kojid
+swupd bundle-add koji
+
+# Create mock folders and permissions
+mkdir -p /etc/mock/koji
+mkdir -p /var/lib/mock
+chown -R root:mock /var/lib/mock
+
+# Setup User Accounts
+useradd -r kojibuilder
+usermod -G mock kojibuilder
+
+# Kojid Configuration Files
+mkdir -p /etc/kojid
+cat > /etc/kojid/kojid.conf <<- EOF
+[kojid]
+sleeptime=5
+maxjobs=16
+topdir=$KOJI_DIR
+workdir=/tmp/koji
+mockdir=/var/lib/mock
+mockuser=kojibuilder
+mockhost=generic-linux-gnu
+user=$KOJI_SLAVE_FQDN
+server=$KOJI_URL/kojihub
+topurl=$KOJI_URL/kojifiles
+use_createrepo_c=True
+allowed_scms=$GIT_FQDN:/packages/*
+cert = $KOJI_PKI_DIR/$KOJI_SLAVE_FQDN.pem
+ca = $KOJI_PKI_DIR/koji_ca_cert.crt
+serverca = $KOJI_PKI_DIR/koji_ca_cert.crt
+EOF
+
+if env | grep -q proxy; then
+	echo "yum_proxy = $https_proxy" >> /etc/kojid/kojid.conf
+	mkdir -p /etc/systemd/system/kojid.service.d
+	cat > /etc/systemd/system/kojid.service.d/00-proxy.conf <<- EOF
+	[Service]
+	Environment=http_proxy=$http_proxy
+	Environment=https_proxy=$https_proxy
+	Environment=no_proxy=$no_proxy
+	EOF
+	systemctl daemon-reload
+fi
+
+systemctl enable --now kojid
--- a/koji-setup/deploy-koji-nfs-client.sh
+++ b/koji-setup/deploy-koji-nfs-client.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (C) 2019 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/globals.sh
+source "$SCRIPT_DIR"/parameters.sh
+
+KOJI_MOUNT_DIR=/mnt/koji
+KOJI_MOUNT_SERVICE="${KOJI_MOUNT_DIR:1}"
+KOJI_MOUNT_SERVICE="${KOJI_MOUNT_SERVICE/\//-}".mount
+mkdir -p /etc/systemd/system
+cat > /etc/systemd/system/"$KOJI_MOUNT_SERVICE" <<- EOF
+[Unit]
+Description=Koji NFS Mount
+After=network.target
+
+[Mount]
+What=$KOJI_MASTER_FQDN:$KOJI_DIR
+Where=$KOJI_MOUNT_DIR
+Type=nfs
+Options=defaults,ro
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl daemon-reload
+systemctl enable --now "$KOJI_MOUNT_SERVICE"
--- a/koji-setup/deploy-koji-nfs-server.sh
+++ b/koji-setup/deploy-koji-nfs-server.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/parameters.sh
+
+swupd bundle-add storage-utils
+
+# Export server directory to be mounted by clients
+echo "$KOJI_DIR $KOJI_SLAVE_FQDN(ro)" >> /etc/exports
+
+systemctl enable --now rpcbind
+systemctl enable --now nfs-server
--- a/koji-setup/deploy-koji.sh
+++ b/koji-setup/deploy-koji.sh
@@ -1,29 +1,17 @@
-#!/usr/bin/env bash
-# Copyright (c) 2018 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

 set -xe
 SCRIPT_DIR="$(dirname "$(realpath "$0")")"
 source "$SCRIPT_DIR"/parameters.sh

-## INSTALL KOJI
-swupd bundle-add koji || true
+# INSTALL KOJI
+swupd bundle-add koji

 ## SETTING UP SSL CERTIFICATES FOR AUTHENTICATION
-KOJI_PKI_DIR=/etc/pki/koji
 mkdir -p "$KOJI_PKI_DIR"/{certs,private}
-RANDFILE=$KOJI_PKI_DIR/.rand
+RANDFILE="$KOJI_PKI_DIR"/.rand
 dd if=/dev/urandom of="$RANDFILE" bs=256 count=1

 # Certificate generation
@@ -62,7 +50,7 @@ commonName              = supplied
 emailAddress            = optional

 [req]
-default_bits            = 1024
+default_bits            = 2048
 default_keyfile         = privkey.pem
 default_md              = sha256
 distinguished_name      = req_distinguished_name
@@ -101,14 +89,18 @@ authorityKeyIdentifier          = keyid:always,issuer:always
 basicConstraints                = CA:true
 EOF

-# Generate CA
+# Generate and trust CA
 touch "$KOJI_PKI_DIR"/index.txt
 echo 01 > "$KOJI_PKI_DIR"/serial
 openssl genrsa -out "$KOJI_PKI_DIR"/private/koji_ca_cert.key 2048
 openssl req -subj "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=koji_ca/CN=$KOJI_MASTER_FQDN" -config "$KOJI_PKI_DIR"/ssl.cnf -new -x509 -days 3650 -key "$KOJI_PKI_DIR"/private/koji_ca_cert.key -out "$KOJI_PKI_DIR"/koji_ca_cert.crt -extensions v3_ca
 mkdir -p /etc/ca-certs/trusted
 cp -a "$KOJI_PKI_DIR"/koji_ca_cert.crt /etc/ca-certs/trusted
-clrtrust generate
+while true; do
+	if clrtrust generate; then
+		break
+	fi
+done

 # Generate the koji component certificates and the admin certificate and generate a PKCS12 user certificate (for web browser)
 cp "$SCRIPT_DIR"/gencert.sh "$KOJI_PKI_DIR"
@@ -121,37 +113,46 @@ popd

 # Copy certificates into ~/.koji for kojiadmin
 useradd kojiadmin
-ADMIN_CERT_DIR=$(getent passwd kojiadmin | cut -d ':' -f 6)/.koji
-mkdir -p "$ADMIN_CERT_DIR"
-cp -f "$KOJI_PKI_DIR"/kojiadmin.pem "$ADMIN_CERT_DIR"/client.crt
-cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_CERT_DIR"/clientca.crt
-cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_CERT_DIR"/serverca.crt
-chown -R kojiadmin:kojiadmin "$ADMIN_CERT_DIR"
+ADMIN_KOJI_DIR="$(echo ~kojiadmin)"/.koji
+mkdir -p "$ADMIN_KOJI_DIR"
+cp -f "$KOJI_PKI_DIR"/kojiadmin.pem "$ADMIN_KOJI_DIR"/client.crt
+cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_KOJI_DIR"/clientca.crt
+cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_KOJI_DIR"/serverca.crt
+chown -R kojiadmin:kojiadmin "$ADMIN_KOJI_DIR"


 ## POSTGRESQL SERVER
 # Initialize PostgreSQL DB
-mkdir -p /var/lib/pgsql
-chown -R postgres:postgres /var/lib/pgsql
-sudo -u postgres initdb --pgdata /var/lib/pgsql/data
+mkdir -p "$POSTGRES_DIR"
+chown -R "$POSTGRES_USER":"$POSTGRES_USER" "$POSTGRES_DIR"
+if [[ "$POSTGRES_DIR" != "$POSTGRES_DEFAULT_DIR" ]]; then
+	if [ "$(ls -A "$POSTGRES_DEFAULT_DIR")" ]; then
+		mv "$POSTGRES_DEFAULT_DIR" "$POSTGRES_DEFAULT_DIR".old
+	else
+		rm -rf "$POSTGRES_DEFAULT_DIR"
+	fi
+	ln -sf "$POSTGRES_DIR" "$POSTGRES_DEFAULT_DIR"
+	chown -h "$POSTGRES_USER":"$POSTGRES_USER" "$POSTGRES_DEFAULT_DIR"
+fi
+sudo -u "$POSTGRES_USER" initdb --pgdata "$POSTGRES_DEFAULT_DIR"/data
 systemctl enable --now postgresql

 # Setup User Accounts
 useradd -r koji

 # Setup PostgreSQL and populate schema
-sudo -u postgres createuser --no-superuser --no-createrole --no-createdb koji
-sudo -u postgres createdb -O koji koji
+sudo -u "$POSTGRES_USER" createuser --no-superuser --no-createrole --no-createdb koji
+sudo -u "$POSTGRES_USER" createdb -O koji koji
 sudo -u koji psql koji koji < /usr/share/doc/koji*/docs/schema.sql

 # Authorize Koji-web and Koji-hub resources
-# TODO: Add authentication with SSL certificates
-cat > /var/lib/pgsql/data/pg_hba.conf <<- EOF
-#TYPE   DATABASE    USER    CIDR-ADDRESS      METHOD
-host    koji        all    127.0.0.1/32       trust
-host    koji        all    ::1/128            trust
-local   koji        all                       trust
+cat > "$POSTGRES_DEFAULT_DIR"/data/pg_hba.conf <<- EOF
+#TYPE    DATABASE    USER    CIDR-ADDRESS    METHOD
+host     koji        all     127.0.0.1/32    trust
+host     koji        all     ::1/128         trust
+local    koji        all                     trust
 EOF
+systemctl reload postgresql

 # Bootstrapping the initial koji admin user into the PostgreSQL database
 # SSL Certificate authentication
@@ -172,17 +173,38 @@ KojiDir = $KOJI_DIR
 DNUsernameComponent = CN
 ProxyDNs = C=$COUNTRY_CODE,ST=$STATE,L=$LOCATION,O=$ORGANIZATION,OU=kojiweb,CN=$KOJI_MASTER_FQDN
 LoginCreatesUser = On
-KojiWebURL = https://$KOJI_MASTER_FQDN/koji
+KojiWebURL = $KOJI_URL/koji
 DisableNotifications = True
 EOF

+mkdir -p /etc/httpd/conf.d
+cat > /etc/httpd/conf.d/kojihub.conf <<- EOF
+Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py
+<Directory "/usr/share/koji-hub">
+    Options ExecCGI
+    SetHandler wsgi-script
+    Require all granted
+</Directory>
+Alias /kojifiles "$KOJI_DIR"
+<Directory "$KOJI_DIR">
+    Options Indexes SymLinksIfOwnerMatch
+    AllowOverride None
+    Require all granted
+</Directory>
+<Location /kojihub/ssllogin>
+    SSLVerifyClient require
+    SSLVerifyDepth 10
+    SSLOptions +StdEnvVars
+</Location>
+EOF
+
 # Koji Web
 mkdir -p /etc/kojiweb
 cat > /etc/kojiweb/web.conf <<- EOF
 [web]
 SiteName = koji
-KojiHubURL = https://$KOJI_MASTER_FQDN/kojihub
-KojiFilesURL = https://$KOJI_MASTER_FQDN/kojifiles
+KojiHubURL = $KOJI_URL/kojihub
+KojiFilesURL = $KOJI_URL/kojifiles
 WebCert = $KOJI_PKI_DIR/kojiweb.pem
 ClientCA = $KOJI_PKI_DIR/koji_ca_cert.crt
 KojiHubCA = $KOJI_PKI_DIR/koji_ca_cert.crt
@@ -192,44 +214,64 @@ LibPath = /usr/share/koji-web/lib
 LiteralFooter = True
 EOF

+mkdir -p /etc/httpd/conf.d
+cat > /etc/httpd/conf.d/kojiweb.conf <<- EOF
+Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py"
+<Directory "/usr/share/koji-web/scripts">
+    Options ExecCGI
+    SetHandler wsgi-script
+    Require all granted
+</Directory>
+Alias /koji-static "/usr/share/koji-web/static"
+<Directory "/usr/share/koji-web/static">
+    Options None
+    AllowOverride None
+    Require all granted
+</Directory>
+EOF
+
 # Koji CLI
-cat > /etc/koji.conf <<- EOF
+cat > "$ADMIN_KOJI_DIR"/config <<- EOF
 [koji]
-server = https://$KOJI_MASTER_FQDN/kojihub
-weburl = https://$KOJI_MASTER_FQDN/koji
-topurl = https://$KOJI_MASTER_FQDN/kojifiles
+server = $KOJI_URL/kojihub
+weburl = $KOJI_URL/koji
+topurl = $KOJI_URL/kojifiles
 topdir = $KOJI_DIR
 cert = ~/.koji/client.crt
 ca = ~/.koji/clientca.crt
 serverca = ~/.koji/serverca.crt
 anon_retry = true
 EOF
-
+chown kojiadmin:kojiadmin "$ADMIN_KOJI_DIR"/config

 ## KOJI APPLICATION HOSTING
 # Koji Filesystem Skeleton
 mkdir -p "$KOJI_DIR"/{packages,repos,work,scratch,repos-dist}
-chown -R httpd:httpd "$KOJI_DIR"
+chown -R "$HTTPD_USER":"$HTTPD_USER" "$KOJI_DIR"

 ## Apache Configuration Files
 mkdir -p /etc/httpd/conf.d
-cat > /etc/httpd/conf.d/koji.conf <<- EOF
-LoadModule ssl_module lib/httpd/modules/mod_ssl.so
+cat > /etc/httpd/conf.d/ssl.conf <<- EOF
+ServerName $KOJI_MASTER_FQDN

-<VirtualHost _default_:80>
-    AllowEncodedSlashes NoDecode
-    RedirectMatch permanent /(.*) https://$KOJI_MASTER_FQDN/$1
-</VirtualHost>
+Listen 443 https
+
+#SSLPassPhraseDialog exec:/usr/libexec/httpd-ssl-pass-dialog
+
+#SSLSessionCache         shmcb:/run/httpd/sslcache(512000)
+
+SSLRandomSeed startup file:/dev/urandom  256
+SSLRandomSeed connect builtin

-Listen 443
 <VirtualHost _default_:443>
-    ServerName $KOJI_MASTER_FQDN
+    ErrorLog /var/log/httpd/ssl_error_log
+    TransferLog /var/log/httpd/ssl_access_log
+    LogLevel warn

    SSLEngine on
    SSLProtocol -all +TLSv1.2
    SSLCipherSuite EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:EDH+aRSA:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!RC4:!DH:!SHA1
    SSLHonorCipherOrder on
-    SSLOptions +StrictRequire +StdEnvVars

    SSLCertificateFile $KOJI_PKI_DIR/kojihub.pem
    SSLCertificateKeyFile $KOJI_PKI_DIR/private/kojihub.key
@@ -238,45 +280,25 @@ Listen 443
    SSLVerifyClient optional
    SSLVerifyDepth 10

-    # Koji Hub
-    Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py
-    <Directory "/usr/share/koji-hub">
-        Options ExecCGI
-        SetHandler wsgi-script
-        Require all granted
+    <Files ~ "\.(cgi|shtml|phtml|php3?)$">
+        SSLOptions +StdEnvVars
+    </Files>
+    <Directory "/var/www/cgi-bin">
+        SSLOptions +StdEnvVars
    </Directory>
-    Alias /kojifiles "/srv/koji/"
-    <Directory "/srv/koji">
-        Options Indexes SymLinksIfOwnerMatch
-        AllowOverride None
-        Require all granted
-    </Directory>
-    <Location /kojihub/ssllogin>
-        SSLVerifyClient require
-    </Location>

-    # Koji Web
-    Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py"
-    <Directory "/usr/share/koji-web/scripts/">
-        Options ExecCGI
-        SetHandler wsgi-script
-        Require all granted
-    </Directory>
-    Alias /koji-static/ "/usr/share/koji-web/static/"
-    <Directory "/usr/share/koji-web/static/">
-        Options None
-        AllowOverride None
-        Require all granted
-    </Directory>
+    CustomLog /var/log/httpd/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"
 </VirtualHost>
 EOF

-## Apache Configuration Files
 mkdir -p /etc/httpd/conf.modules.d
 cat > /etc/httpd/conf.modules.d/wsgi.conf <<- EOF
 LoadModule wsgi_module lib/python2.7/site-packages/mod_wsgi/server/mod_wsgi-py27.so
 WSGISocketPrefix /run/httpd/wsgi
 EOF
+cat > /etc/httpd/conf.modules.d/ssl.conf <<- EOF
+LoadModule ssl_module lib/httpd/modules/mod_ssl.so
+EOF

 systemctl enable --now httpd

@@ -287,7 +309,7 @@ sudo -u kojiadmin koji moshimoshi

 ## KOJI DAEMON - BUILDER
 # Add the host entry for the koji builder to the database
-sudo -u kojiadmin koji add-host "$KOJI_SLAVE_FQDN" i386 x86_64
+sudo -u kojiadmin koji add-host "$KOJI_SLAVE_FQDN" "$RPM_ARCH"

 # Add the host to the createrepo channel
 sudo -u kojiadmin koji add-host-to-channel "$KOJI_SLAVE_FQDN" createrepo
@@ -300,50 +322,10 @@ pushd "$KOJI_PKI_DIR"
 ./gencert.sh "$KOJI_SLAVE_FQDN" "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/CN=$KOJI_SLAVE_FQDN"
 popd

-# Create mock folders and permissions
-mkdir -p /etc/mock/koji
-mkdir -p /var/lib/mock
-chown -R root:mock /var/lib/mock
-
-# Setup User Accounts
-useradd -r kojibuilder
-usermod -G mock kojibuilder
-
-# Kojid Configuration Files
-mkdir -p /etc/kojid
-cat > /etc/kojid/kojid.conf <<- EOF
-[kojid]
-sleeptime=5
-maxjobs=16
-topdir=$KOJI_DIR
-workdir=/tmp/koji
-mockdir=/var/lib/mock
-mockuser=kojibuilder
-mockhost=generic-linux-gnu
-user=$KOJI_SLAVE_FQDN
-server=https://$KOJI_MASTER_FQDN/kojihub
-topurl=https://$KOJI_MASTER_FQDN/kojifiles
-use_createrepo_c=True
-allowed_scms=$CGIT_FQDN:/packages/*
-cert = $KOJI_PKI_DIR/$KOJI_SLAVE_FQDN.pem
-ca = $KOJI_PKI_DIR/koji_ca_cert.crt
-serverca = $KOJI_PKI_DIR/koji_ca_cert.crt
-EOF
-
-if env | grep -q proxy; then
-	echo "yum_proxy = $https_proxy" >> /etc/kojid/kojid.conf
-	mkdir -p /etc/systemd/system/kojid.service.d
-	cat > /etc/systemd/system/kojid.service.d/00-proxy.conf <<- EOF
-	[Service]
-	Environment=http_proxy=$http_proxy
-	Environment=https_proxy=$https_proxy
-	Environment=no_proxy=$no_proxy
-	EOF
-	systemctl daemon-reload
+if [[ "$KOJI_SLAVE_FQDN" = "$KOJI_MASTER_FQDN" ]]; then
+	"$SCRIPT_DIR"/deploy-koji-builder.sh
 fi

-systemctl enable --now kojid
-

 ## KOJIRA - DNF|YUM REPOSITORY CREATION AND MAINTENANCE
 # Add the user entry for the kojira user
@@ -354,7 +336,7 @@ sudo -u kojiadmin koji grant-permission repo kojira
 mkdir -p /etc/kojira
 cat > /etc/kojira/kojira.conf <<- EOF
 [kojira]
-server=https://$KOJI_MASTER_FQDN/kojihub
+server=$KOJI_URL/kojihub
 topdir=$KOJI_DIR
 logfile=/var/log/kojira.log
 with_src=no
--- a/koji-setup/deploy-mash.sh
+++ b/koji-setup/deploy-mash.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/parameters.sh
+
+swupd bundle-add koji
+
+mkdir -p "$MASH_DIR"
+chown -R kojiadmin:kojiadmin "$MASH_DIR"
+mkdir -p "$HTTPD_DOCUMENT_ROOT"
+MASH_LINK="$HTTPD_DOCUMENT_ROOT"/"$(basename "$MASH_DIR")"
+ln -sf "$MASH_DIR"/latest "$MASH_LINK"
+chown -h kojiadmin:kojiadmin "$MASH_LINK"
+usermod -a -G kojiadmin "$HTTPD_USER"
+# Required because Clear is stateless, and mash is run as a non-elevated user
+mkdir -p /var/cache/mash
+rpm --initdb
+
+mkdir -p /etc/mash
+cat > /etc/mash/mash.conf <<- EOF
+[defaults]
+configdir = /etc/mash
+buildhost = $KOJI_URL/kojihub
+repodir = file://$KOJI_DIR
+use_sqlite = True
+use_repoview = False
+EOF
+cat > /etc/mash/clear.mash <<- EOF
+[clear]
+rpm_path = %(arch)s/os/Packages
+repodata_path = %(arch)s/os/
+source_path = source/SRPMS
+debuginfo = True
+multilib = False
+multilib_method = devel
+tag = dist-$TAG_NAME
+inherit = True
+strict_keys = False
+arches = $RPM_ARCH
+EOF
+
+mkdir -p "$MASH_SCRIPT_DIR"
+cp -f "$SCRIPT_DIR"/mash.sh "$MASH_SCRIPT_DIR"
+mkdir -p /etc/systemd/system
+cat > /etc/systemd/system/mash.service <<- EOF
+[Unit]
+Description=Mash script to loop local repository creation for local image builds
+
+[Service]
+User=kojiadmin
+Group=kojiadmin
+ExecStart=$MASH_SCRIPT_DIR/mash.sh
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+systemctl daemon-reload
+systemctl enable --now mash
--- a/koji-setup/deploy-upstreams.sh
+++ b/koji-setup/deploy-upstreams.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+source "$SCRIPT_DIR"/parameters.sh
+
+mkdir -p "$UPSTREAMS_DIR"
+chown -R "$GIT_USER":"$GIT_USER" "$UPSTREAMS_DIR"
+mkdir -p "$HTTPD_DOCUMENT_ROOT"
+UPSTREAMS_LINK="$HTTPD_DOCUMENT_ROOT"/"$(basename "$UPSTREAMS_DIR")"
+ln -sf "$UPSTREAMS_DIR" "$UPSTREAMS_LINK"
+chown -h "$GIT_USER":"$GIT_USER" "$UPSTREAMS_LINK"
+usermod -a -G "$GIT_USER" "$HTTPD_USER"
--- a/koji-setup/gencert.sh
+++ b/koji-setup/gencert.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
-# user is equal to parameter one or the first argument when you actually
-# run the script
-user=$1
-# subject is equal to parameter two or the second argument when you actually
-# run the script
-subject="$2"
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

-openssl genrsa -out private/${user}.key 2048
-if [ -z "$subject" ]; then
-	openssl req -config ssl.cnf -new -nodes -out certs/${user}.csr -key private/${user}.key
+KOJI_USER="$1"
+CERT_SUBJECT="$2"
+
+openssl genrsa -out private/"$KOJI_USER".key 2048
+if [ -z "$CERT_SUBJECT" ]; then
+	openssl req -config ssl.cnf -new -nodes -out certs/"$KOJI_USER".csr -key private/"$KOJI_USER".key
 else
-	openssl req -subj "$subject" -config ssl.cnf -new -nodes -out certs/${user}.csr -key private/${user}.key
+	openssl req -subj "$CERT_SUBJECT" -config ssl.cnf -new -nodes -out certs/"$KOJI_USER".csr -key private/"$KOJI_USER".key
 fi
-openssl ca -batch -config ssl.cnf -keyfile private/koji_ca_cert.key -cert koji_ca_cert.crt -out certs/${user}.crt -outdir certs -infiles certs/${user}.csr
-cat certs/${user}.crt private/${user}.key > ${user}.pem
-openssl pkcs12 -export -inkey private/${user}.key -in certs/${user}.crt -CAfile koji_ca_cert.crt -out certs/${user}_browser_cert.p12 -passout pass:
+openssl ca -batch -config ssl.cnf -keyfile private/koji_ca_cert.key -cert koji_ca_cert.crt -out certs/"$KOJI_USER".crt -outdir certs -infiles certs/"$KOJI_USER".csr
+cat certs/"$KOJI_USER".crt private/"$KOJI_USER".key > "$KOJI_USER".pem
+# Browser certificate is not password-protected, ask users to change their password
+openssl pkcs12 -export -inkey private/"$KOJI_USER".key -in certs/"$KOJI_USER".crt -CAfile koji_ca_cert.crt -out certs/"$KOJI_USER"_browser_cert.p12 -passout pass:
--- a/koji-setup/mash.sh
+++ b/koji-setup/mash.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+set -xe
+if [[ -e /etc/profile.d/proxy.sh ]]; then
+	source /etc/profile.d/proxy.sh
+fi
+
+KOJI_HOME=/srv/koji
+MASH_HOME=/srv/mash
+MASH_TRACKER_FILE="$MASH_HOME"/latest-mash-build
+MASH_DIR="$MASH_HOME"/latest
+MASH_DIR_OLD="$MASH_DIR".old
+MASH_DIR_NEW="$MASH_DIR".new
+
+if [[ -e "$MASH_TRACKER_FILE" ]]; then
+	MASH_BUILD_NUM="$(< "$MASH_TRACKER_FILE")"
+else
+	MASH_BUILD_NUM=0
+fi
+KOJI_BUILD_NUM="$(basename "$(realpath "$KOJI_HOME"/repos/dist-clear-build/latest/)")"
+if [[ "$MASH_BUILD_NUM" -ne "$KOJI_BUILD_NUM" ]]; then
+	COMPS_FILE=$(mktemp)
+	koji show-groups --comps dist-clear-build > "$COMPS_FILE"
+	rm -rf "$MASH_DIR_NEW"
+	mash --outputdir="$MASH_DIR_NEW" --compsfile="$COMPS_FILE" clear
+	rm -f "$COMPS_FILE"
+	dnf -q --repofrompath=mash,file://"$MASH_DIR_NEW"/clear/x86_64/os repoquery -a --queryformat="%{NAME}\t%{VERSION}\t%{RELEASE}" | sort > "$MASH_DIR_NEW"/clear/x86_64/packages-os &>/dev/null
+	dnf -q --repofrompath=mash,file://"$MASH_DIR_NEW"/clear/x86_64/debug repoquery -a --queryformat="%{NAME}\t%{VERSION}\t%{RELEASE}" | sort > "$MASH_DIR_NEW"/clear/x86_64/packages-debug &>/dev/null
+	dnf -q --repofrompath=mash,file://"$MASH_DIR_NEW"/clear/source/SRPMS repoquery -a --queryformat="%{NAME}\t%{VERSION}\t%{RELEASE}" | sort > "$MASH_DIR_NEW"/clear/source/packages-SRPMS &>/dev/null
+	if [[ -e "$MASH_DIR" ]]; then
+		mv "$MASH_DIR" "$MASH_DIR_OLD"
+	fi
+	mv "$MASH_DIR_NEW" "$MASH_DIR"
+	rm -rf "$MASH_DIR_OLD"
+	echo "$KOJI_BUILD_NUM" > "$MASH_TRACKER_FILE"
+fi
--- a/koji-setup/parameters.sh
+++ b/koji-setup/parameters.sh
@@ -1,12 +1,51 @@
+#!/bin/bash
+# Copyright (C) 2018 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+## KOJI RPM BUILD AND TRACKER
+export KOJI_DIR=/srv/koji
+export KOJI_MASTER_FQDN="$(hostname -f)"
+export KOJI_URL=https://"$KOJI_MASTER_FQDN"
+export KOJI_MOUNT_DIR=/mnt/koji
+export KOJI_SLAVE_FQDN="$KOJI_MASTER_FQDN"
+export KOJID_CAPACITY=16
+export TAG_NAME=clear
+# Use for koji SSL certificates
+export KOJI_PKI_DIR=/etc/pki/koji
 export COUNTRY_CODE=US
 export STATE=Oregon
 export LOCATION=Hillsboro
-export ORGANIZATION='Example org'
-export ORG_UNIT='Example unit'
-export KOJI_MASTER_FQDN="$(hostname -f)"
-export KOJI_SLAVE_FQDN="$KOJI_MASTER_FQDN"
-export KOJI_DIR=/srv/koji
-export CGIT_FQDN="$KOJI_MASTER_FQDN"
+export ORGANIZATION='Example Organization'
+export ORG_UNIT='Example Unit'
+# Use for importing existing RPMs
+export RPM_ARCH='x86_64'
+export SRC_RPM_DIR=
+export BIN_RPM_DIR=
+export DEBUG_RPM_DIR=
+# Comment the following if supplying all RPMs as an upstream and not a downstream
 export EXTERNAL_REPO=https://cdn.download.clearlinux.org/releases/"$(curl https://download.clearlinux.org/latest)"/clear/\$arch/os/
-export TAG_NAME=clear
-export KOJID_CAPACITY=16
+
+## POSTGRESQL DATABASE
+export POSTGRES_DIR=/srv/pgsql
+export POSTGRES_DEFAULT_DIR=/var/lib/pgsql
+export POSTGRES_USER=postgres
+
+## GIT REPOSITORIES
+export GIT_DIR=/srv/gitolite
+export GIT_DEFAULT_DIR=/var/lib/gitolite
+export GIT_USER=gitolite
+export GIT_FQDN="$KOJI_MASTER_FQDN"
+export IS_ANONYMOUS_GIT_NEEDED=false
+export GITOLITE_PUB_KEY=''
+
+## APACHE WEB SERVER
+export HTTPD_USER=httpd
+# Autoindexing should occur in this directory by default
+export HTTPD_DOCUMENT_ROOT=/var/www/html
+
+## UPSTREAMS CACHE
+export UPSTREAMS_DIR=/srv/upstreams
+
+## MASH RPMS
+export MASH_DIR=/srv/mash
+export MASH_SCRIPT_DIR=/usr/local/bin