From a198b2d0a0cf8a875d4ea8d95b52f12aeb7bd62c Mon Sep 17 00:00:00 2001
From: Patrick McCarty <patrick.mccarty@intel.com>
Date: Mon, 18 May 2020 13:55:19 -0700
Subject: [PATCH] Make local repo support more parallel safe

For commands that modify contents of $(TOPLVL)/repo, wrap them with
`flock`, using a lock file `repo.lock` within the repo directory. This
ensures that the repo directory tree is not otherwise modified while
each command is running.

To keep the implementation simple, I opted to not wrap entire makefile
targets with a lock (e.g. lock the entire operation of `make repoadd`).
If users need that level of control, they can implement locking in a
separate wrapper script.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>
---
 Makefile.common   | 69 +++++++++++++++++++----------------------------
 Makefile.shared   | 48 +++++++++++++++++++++------------
 Makefile.toplevel |  7 +----
 3 files changed, 59 insertions(+), 65 deletions(-)

diff --git a/Makefile.common b/Makefile.common
index 904cba1..e616e15 100644
--- a/Makefile.common
+++ b/Makefile.common
@@ -349,11 +349,6 @@ rekoji-nowait:
 logcheck:
 	@perl $(TOPLVL)/projects/common/logcheck.pl results/build.log
 
-$(TOPLVL)/repo:
-	@echo "Creating local RPM repository $(TOPLVL)/repo"
-	mkdir $(TOPLVL)/repo
-	$(MAKE) localrepocreate
-
 #help repoadd: Adds locally-built RPMs for this package to the local RPM
 #help repository. If this repository does not yet exist, it is created and
 #help enabled.
@@ -361,22 +356,16 @@ repoadd: $(TOPLVL)/repo
 	@if [ -z "${LATEST_RPMS}" ]; then \
 		echo "No rpms found in rpms/ directory."; \
 		exit 1; \
-	fi; \
-	if [ -f .repo-index ]; then \
-		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
-			echo "-$$r"; \
-			rm -f $(TOPLVL)/repo/$$r; \
-		done; \
-		rm .repo-index; \
-	fi; \
-	for r in ${LATEST_RPMS}; do \
-		echo "+$$r"; \
-		echo $$(basename $$r) >> .repo-index; \
-		ln -f $$r $(TOPLVL)/repo/; \
-	done
-	$(MAKE) localrepocreate
-	$(call localrepoenable,${PM_CONF},${MOCK_CONF})
+	fi
+	$(MAKE) -s repodel NO_CREATEREPO=1
+	@echo "Adding new rpms:"; \
+	for rpm in ${LATEST_RPMS}; do \
+		echo "+$${rpm#rpms/}"; \
+		echo "$${rpm#rpms/}" >> .repo-index; \
+	done; \
+	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
+	$(MAKE) -s localrepocreate
+	$(MAKE) -s localrepoenable
 
 #help repostage: Stages locally-built RPMs for this package to the local RPM
 #help directory. If this directory does not yet exist, it is created and
@@ -385,37 +374,33 @@ repostage: $(TOPLVL)/repo
 	@if [ -z "${LATEST_RPMS}" ]; then \
 		echo "No rpms found in rpms/ directory."; \
 		exit 1; \
-	fi; \
-	if [ -f .repo-index ]; then \
-		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
-			echo "-$$r"; \
-			rm -f $(TOPLVL)/repo/$$r; \
-		done; \
-		rm .repo-index; \
-	fi; \
-	for r in ${LATEST_RPMS}; do \
-		echo "+$$r"; \
-		echo $$(basename $$r) >> .repo-index; \
-		ln -f $$r $(TOPLVL)/repo/; \
-	done
+	fi
+	$(MAKE) -s repodel NO_CREATEREPO=1
+	@for r in ${LATEST_RPMS}; do \
+		echo "+$${rpm#rpms/}"; \
+		echo "$${rpm#rpms/}" >> .repo-index; \
+	done; \
+	flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
 
 #help repodel: Removes RPMs from the local RPM repository that were
 #help previously added by 'make repoadd' for this package.
-repodel:
+repodel: $(TOPLVL)/repo
 	@if [ -f .repo-index ]; then \
 		echo "Cleaning old rpms:"; \
-		for r in `cat .repo-index`; do \
-			echo "-$$r"; \
-			rm -f $(TOPLVL)/repo/$$r; \
-		done; \
+		mapfile -t rpms < <(sed 's|.*|$</&|' .repo-index); \
+		for rpm in "$${rpms[@]}"; do \
+			echo "-$${rpm#$</}"; \
+		done < .repo-index; \
+		flock $</repo.lock rm -f "$${rpms[@]}"; \
 		rm .repo-index; \
-		$(MAKE) localrepocreate; \
+		if [ -z "$$NO_CREATEREPO" ]; then \
+			$(MAKE) -s localrepocreate; \
+		fi; \
 	fi
 
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;
 
 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;
diff --git a/Makefile.shared b/Makefile.shared
index e8a4d1b..ae6cb61 100644
--- a/Makefile.shared
+++ b/Makefile.shared
@@ -93,34 +93,48 @@ $(MAKE) clone_$(1) || { \
 }
 endef
 
+$(TOPLVL)/repo:
+	@echo "Creating local RPM repository $(TOPLVL)/repo"
+	mkdir $(TOPLVL)/repo
 
 # Enables the local RPM repo by installing package manager and Mock configs.
 # First argument is the path to the package manager config, and the second
 # argument is the path to the mock config. Each of these config files is
 # augmented for local repo support.
-define localrepoenable
-	@cp $(1) $(TOPLVL)/repo/yum.conf
-	@printf "\n\n[local]\nname=Local\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/yum.conf
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/yum.conf
-	@cp $(2) $(TOPLVL)/repo/clear.cfg
-	@printf "\n\n[localrepo]\nname=Local repo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $(TOPLVL)/repo/clear.cfg
-	@printf "baseurl=file://$$(realpath $(TOPLVL))/repo/\n\n" >> $(TOPLVL)/repo/clear.cfg
-	@sed -i '/^"""/d; $$a"""' $(TOPLVL)/repo/clear.cfg
-endef
+localrepoenable: $(TOPLVL)/repo
+	@pm_new=$$(mktemp -p $< yum.conf.XXXXXXX); \
+	pm_orig=$</yum.conf; \
+	mock_new=$$(mktemp -p $< clear.cfg.XXXXXXX); \
+	mock_orig=$</clear.cfg; \
+	cp ${PM_CONF} $$pm_new; \
+	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$pm_new; \
+	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$pm_new; \
+	if ! cmp -s $$pm_new $$pm_orig; then \
+		flock $</repo.lock mv $$pm_new $$pm_orig; \
+	fi; \
+	rm -f $$pm_new; \
+	cp ${MOCK_CONF} $$mock_new; \
+	sed -i '$$d' $$mock_new; \
+	printf "\n\n[localrepo]\nname=localrepo\nfailovermethod=priority\nenabled=1\ngpgcheck=0\npriority=1\n" >> $$mock_new; \
+	printf "baseurl=file://$$(realpath $<)/\n\n" >> $$mock_new; \
+	sed -i '$$a"""' $$mock_new; \
+	if ! cmp -s $$mock_new $$mock_orig; then \
+		flock $</repo.lock mv $$mock_new $$mock_orig; \
+	fi; \
+	rm -f $$mock_new
 
 # Disables the local RPM repo by removing the installed package manager and
 # Mock configuration files.
-localrepodisable:
-	@rm -f $(TOPLVL)/repo/yum.conf
-	@rm -f $(TOPLVL)/repo/clear.cfg
+localrepodisable: $(TOPLVL)/repo
+	flock $</repo.lock rm -f $</yum.conf $</clear.cfg
 
-# Runs createrepo_c (or createrepo) for the local RPM repo
-localrepocreate:
-	( cd $(TOPLVL)/repo ; if type createrepo_c 1>/dev/null 2>&1; then createrepo_c .; else createrepo .; fi );
+# Runs createrepo_c for the local RPM repo
+localrepocreate: $(TOPLVL)/repo
+	cd $< && flock repo.lock createrepo_c .
 
 # Remove all RPMs from the local RPM repo
-localrepoclean:
-	@find $(TOPLVL)/repo -maxdepth 1 -name '*.rpm' -delete
+localrepoclean: $(TOPLVL)/repo
+	flock $</repo.lock find $< -maxdepth 1 -name '*.rpm' -delete
 
 # Summarizes the local RPM repo status
 localrepostatus:
diff --git a/Makefile.toplevel b/Makefile.toplevel
index 2405ae6..a7ecc0b 100644
--- a/Makefile.toplevel
+++ b/Makefile.toplevel
@@ -290,14 +290,9 @@ autospecnewgo: preautospecnew-checks localreponotice
 		exit 1; \
 	fi
 
-$(TOPLVL)/repo:
-	@echo "Creating local RPM repository $(TOPLVL)/repo"
-	mkdir $(TOPLVL)/repo
-	$(MAKE) localrepocreate
-
 #help repoenable: Enables the local RPM repository for use with Yum/DNF and
 #help Mock. If this repository does not yet exist, it is created.
-repoenable: $(TOPLVL)/repo ; $(call localrepoenable,${PM_CONF},${MOCK_CONF})
+repoenable: localrepoenable ;
 
 #help repodisable: Disables the local RPM repository.
 repodisable: localrepodisable ;