go2spec-script/sync_waiting_packages.sh

#!/usr/bin/env bash

# grep -ohP 'go\(\K[^)]+' */*.spec | sort -u > waiting

set -u

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXIST_FILE="${SCRIPT_DIR}/exist"
WAITING_FILE="${SCRIPT_DIR}/waiting"
DOWNLOADED_FILE="${SCRIPT_DIR}/downloaded"
GO2SPEC_BIN="${SCRIPT_DIR}/go2spec"
DRY_RUN=0

if [[ "${1:-}" == "--dry-run" ]]; then
  DRY_RUN=1
  echo "[dry-run] enabled: no files will be changed, no download command will run"
elif [[ "${1:-}" != "" ]]; then
  echo "usage: $0 [--dry-run]" >&2
  exit 1
fi

if [[ ! -f "${EXIST_FILE}" ]]; then
  echo "missing file: ${EXIST_FILE}" >&2
  exit 1
fi

if [[ ! -f "${WAITING_FILE}" ]]; then
  echo "missing file: ${WAITING_FILE}" >&2
  exit 1
fi

if [[ ! -f "${DOWNLOADED_FILE}" ]]; then
  : > "${DOWNLOADED_FILE}"
fi

if [[ ! -x "${GO2SPEC_BIN}" ]]; then
  echo "missing executable: ${GO2SPEC_BIN}" >&2
  exit 1
fi

append_if_missing() {
  local value="$1"
  local file="$2"
  if grep -Fxq "${value}" "${file}"; then
    return 1
  fi
  echo "${value}" >> "${file}"
  return 0
}

remove_one_exact_line() {
  local value="$1"
  local file="$2"
  local tmp_file
  if ! grep -Fxq "${value}" "${file}"; then
    return 1
  fi
  tmp_file="$(mktemp)"
  awk -v target="${value}" '
    BEGIN { removed = 0 }
    {
      if (!removed && $0 == target) {
        removed = 1
        next
      }
      print
    }
  ' "${file}" > "${tmp_file}"
  mv "${tmp_file}" "${file}"
  return 0
}

trim_line() {
  local line="$1"
  line="${line#"${line%%[![:space:]]*}"}"
  line="${line%"${line##*[![:space:]]}"}"
  printf '%s' "${line}"
}

escape_pkg_name() {
  local pkg="$1"
  # github.com -> github
  pkg="${pkg/#github.com\//github/}"
  # convert path separators, dots, underscores to dashes
  pkg="${pkg//\//-}"
  pkg="${pkg//./-}"
  pkg="${pkg//_/-}"
  printf 'go-%s' "${pkg}"
}

processed_count=0
downloaded_count=0
skipped_count=0
failed_count=0

print_summary() {
  echo
  echo "done"
  echo "processed:  ${processed_count}"
  echo "downloaded: ${downloaded_count}"
  echo "skipped:    ${skipped_count}"
  echo "failed:     ${failed_count}"
}

handle_interrupt() {
  echo
  echo "[interrupted] received Ctrl+C, printing current summary"
  if [[ -n "${waiting_snapshot:-}" && -f "${waiting_snapshot}" ]]; then
    rm -f "${waiting_snapshot}"
  fi
  print_summary
  exit 130
}

trap handle_interrupt INT

while [[ -s "${WAITING_FILE}" ]]; do
  round_progress=0
  waiting_snapshot="$(mktemp)"
  cp "${WAITING_FILE}" "${waiting_snapshot}"

  while IFS= read -r raw_pkg || [[ -n "${raw_pkg}" ]]; do
    pkg="$(trim_line "${raw_pkg}")"

    if [[ -z "${pkg}" ]]; then
      if [[ "${DRY_RUN}" -eq 1 ]]; then
        echo "[step7][dry-run] would remove blank line from waiting"
      else
        if remove_one_exact_line "${raw_pkg}" "${WAITING_FILE}"; then
          round_progress=1
          echo "[step7] removed blank line from waiting"
        fi
      fi
      continue
    fi

    processed_count=$((processed_count + 1))
    escaped_pkg="$(escape_pkg_name "${pkg}")"

    if grep -Fxq "${escaped_pkg}" "${EXIST_FILE}"; then
      skipped_count=$((skipped_count + 1))
      echo "[skip] ${pkg} (already in exist as ${escaped_pkg})"
      if [[ "${DRY_RUN}" -eq 1 ]]; then
        echo "[step7][dry-run] would remove from waiting: ${pkg} (reason: already exists)"
      else
        if remove_one_exact_line "${raw_pkg}" "${WAITING_FILE}"; then
          round_progress=1
          echo "[step7] removed from waiting: ${pkg} (reason: already exists)"
        else
          echo "[step7] unable to remove from waiting (line mismatch): ${pkg}"
        fi
      fi
      continue
    fi

    if [[ "${DRY_RUN}" -eq 1 ]]; then
      echo "[step5][dry-run] would run: ${GO2SPEC_BIN} pack -allow_unknown_hoster ${pkg}"
      echo "[step6][dry-run] would append exist <= ${escaped_pkg}"
      echo "[step6][dry-run] would append downloaded <= ${pkg}"
      echo "[step7][dry-run] would remove from waiting: ${pkg}"
      downloaded_count=$((downloaded_count + 1))
      continue
    fi

    echo "[step5] run download command for: ${pkg}"
    if "${GO2SPEC_BIN}" pack -allow_unknown_hoster "${pkg}"; then
      downloaded_count=$((downloaded_count + 1))
      if append_if_missing "${escaped_pkg}" "${EXIST_FILE}"; then
        echo "[step6] exist appended: ${escaped_pkg}"
      else
        echo "[step6] exist already had: ${escaped_pkg}"
      fi
      if append_if_missing "${pkg}" "${DOWNLOADED_FILE}"; then
        echo "[step6] downloaded appended: ${pkg}"
      else
        echo "[step6] downloaded already had: ${pkg}"
      fi
      if remove_one_exact_line "${raw_pkg}" "${WAITING_FILE}"; then
        round_progress=1
        echo "[step7] removed from waiting: ${pkg}"
      else
        echo "[step7] unable to remove from waiting (line mismatch): ${pkg}"
      fi
      echo "[ok] ${pkg}"
    else
      echo "[retry] first attempt failed for ${pkg}, removing ~/go and retrying"
      rm -rf "${HOME}/go"
      echo "[step5] retry download command for: ${pkg}"
      if "${GO2SPEC_BIN}" pack -allow_unknown_hoster "${pkg}"; then
        downloaded_count=$((downloaded_count + 1))
        if append_if_missing "${escaped_pkg}" "${EXIST_FILE}"; then
          echo "[step6] exist appended: ${escaped_pkg}"
        else
          echo "[step6] exist already had: ${escaped_pkg}"
        fi
        if append_if_missing "${pkg}" "${DOWNLOADED_FILE}"; then
          echo "[step6] downloaded appended: ${pkg}"
        else
          echo "[step6] downloaded already had: ${pkg}"
        fi
        if remove_one_exact_line "${raw_pkg}" "${WAITING_FILE}"; then
          round_progress=1
          echo "[step7] removed from waiting: ${pkg}"
        else
          echo "[step7] unable to remove from waiting (line mismatch): ${pkg}"
        fi
        echo "[ok-after-retry] ${pkg}"
      else
        failed_count=$((failed_count + 1))
        echo "[step7] keep in waiting: ${pkg} (reason: download failed twice)"
        echo "[failed] ${pkg}" >&2
      fi
    fi
  done < "${waiting_snapshot}"

  rm -f "${waiting_snapshot}"
  waiting_snapshot=""

  if [[ "${DRY_RUN}" -eq 1 ]]; then
    break
  fi

  if [[ "${round_progress}" -eq 0 ]]; then
    echo "[stop] no progress in this round; remaining packages stay in waiting"
    break
  fi
done

print_summary