In-cluster registry seeding fails due to insufficient memory

Known issue and workaround for in-cluster registry seeding failures caused by insufficient memory in Automation Suite.

Description

Seeding the in-cluster registry requires loading large image archives into memory. On nodes with limited RAM, this can cause the seeding process to hang or terminate unexpectedly.

Solution

To address the issue, use the following script to seed the in-cluster registry:

#!/bin/bash
set -euo pipefail
[[ $# -lt 1 ]] && { echo "Usage: $0 <bundle>"; exit 1; }
BUNDLE=$1
OFFLINE_FOLDER="/uipath" # don't add / at the end
DOCKER_REGISTRY_NODEPORT=30071
function error() {
  echo -e "\e[0;31m[ERROR][$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
  exit 1
}
function info() {
  echo "[INFO] [$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*"
}
function warn() {
  echo -e "\e[0;33m[WARN] [$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
}
function configure_podman() {
  local podman_folder="$1"
  local podman_config_path="/etc/containers/storage.conf"
  local podman_rootless_path="${podman_folder}/podman_rootless"
  local podman_rootfull_path="${podman_folder}/podman_rootfull"
  if [[ -n "${podman_folder}" ]]; then
    info "Configuring podman to use ${podman_folder}"
    mkdir -p "${podman_rootless_path}" || exit 1
    mkdir -p "${podman_rootfull_path}" || exit 1
    # update the graphroot config line
    sed -i "s|^\(graphroot[[:space:]]=\).*|\1\"${podman_rootfull_path}\"|" "${podman_config_path}"
    # uncomment and update the rootless path config line
    sed -i "s|^#[[:space:]]*\(rootless_storage_path[[:space:]]*=\).*|\1\"${podman_rootless_path}\"|" "${podman_config_path}"
  fi
  # uncomment the mount program
  sed -i "/^#mount_program[[:space:]]*=/s|^#||g" "${podman_config_path}"
}
function get_registry_addresses() {
  local namespace="docker-registry"
  local lh_registry=1
  local server_node_count
  lh_registry=$(kubectl get deployment -n "${namespace}" docker-registry --ignore-not-found --no-headers | wc -l)
  server_node_count=$(kubectl get nodes -l "node-role.kubernetes.io/master=true" --no-headers | wc -l)
  IFS=" " read -r -a registry_addresses <<<"$(kubectl get pods -n ${namespace} -o jsonpath="{.items[*].status.podIP}" -l app=docker-registry)"
  if [[ "${lh_registry}" -eq 1 ]]; then
    [[ "${#registry_addresses[@]}" -eq 1 ]] || error "Failed to get address of registry pod."
  else
    [[ "${#registry_addresses[@]}" -eq "${server_node_count}" ]] || error "Failed to get address of all registry pods."
  fi
  echo "${registry_addresses[@]}"
}
function import_sf_images() {
  local sf_folder="${1}"
  local podman_folder="${2}"
  local image_inventory="${sf_folder}/sf-images/image-tags.txt"
  local image_tar="${sf_folder}/sf-images/images.tar"
  info "Populating Docker Registry"
  if [[ -z "${podman_folder}" ]]; then
    podman_folder="${sf_folder}"
  fi
  [[ -f "${image_inventory}" ]] || error "Missing ${image_inventory}"
  local all_images_present=1
  while read -r image_name; do
    podman image exists "localhost:${DOCKER_REGISTRY_NODEPORT}/${image_name}" || all_images_present=0
  done <"${image_inventory}"
  mkdir -p "${podman_folder}/tmp"
  info "Executing podman load with podman_folder=${podman_folder}"
  # inline ENV for https://github.com/containers/podman/pull/5412 and https://github.com/containers/podman/issues/5960
  export TMPDIR="${podman_folder}/tmp"
  if ((all_images_present != 1)); then
    info "Loading image tar ${image_tar}"
    podman load -i "${image_tar}" || error "Failed to load image"
  else
    info "All images present. Skipping podman load"
  fi
  registry_addresses=($(get_registry_addresses))
  while read -r image_name; do
    for ip in "${registry_addresses[@]}"; do
      local try=0
      local maxtry=10
      local status="notready"
      podman tag "localhost:${DOCKER_REGISTRY_NODEPORT}/${image_name}" "${ip}:5000/${image_name}"
      podman login "${ip}:5000" --tls-verify=false
      while [[ ${status} == "notready" ]] && ((try != maxtry)); do
        try=$((try + 1))
        info "Pushing image ${image_name}...try ${try}/${maxtry}" && sleep 5
        podman push "${ip}:5000/${image_name}" \
          --tls-verify=false  &&
          status="ready"
        [[ ${status} == "ready" ]] || sleep 60
      done
      [[ ${status} == "ready" ]] || error "Failed to push image $image_name"
    done
  done <"${image_inventory}"
  info "Populated Docker Registry"
}
tmp_offline_folder="$(mktemp -d -p "${OFFLINE_FOLDER}" -t offline-bundle.XXXXXXXXXX)"
tar -xzf "${BUNDLE}" -C "${tmp_offline_folder}"
podman_folder="$OFFLINE_FOLDER/lib"
mkdir -p $podman_folder
configure_podman "$podman_folder"
import_sf_images "${tmp_offline_folder}" "$podman_folder"

To run the script, use the following command:

./script.sh <path-to-bundle.tar.gz>