Degraded MongoDB or Business Applications After Cluster Restore

Description

Occasionally, after cluster restore/rollback to version 2022.4.x or 2021.10.x, an issue causes MongoDB or business applications (Apps) pods to get stuck in the initial state. This happens because the volume required for attaching the PVC to a pod is missing.

Solution

1. Verify if the problem is indeed related to MongoDB volume attachment issue:

   # fetch all mongodb pods kubectl -n mongodb get pods #describe pods stuck in init state #kubectl -n mongodb describe pods mongodb-replica-set-<replica index number> kubectl -n mongodb describe pods mongodb-replica-set-0

   If the issue is related to the MongoDB volume attachment, the following events are displayed:

   Events:
     Type     Reason              Age                   From                     Message
     ----                   ----                  ----                     
     Warning  FailedAttachVolume  3m9s (x65 over 133m)  attachdetach-controller  AttachVolume.Attach failed for volume "pvc-66897693-e52d-4b89-aac6-ca0cc5ae9e07" : rpc error: code = Aborted desc = volume pvc-66897693-e52d-4b89-aac6-ca0cc5ae9e07 is not ready for workloads
     Warning  FailedMount         103s (x50 over 112m)  kubelet                  (combined from similar events): Unable to attach or mount volumes: unmounted volumes=[logs-volume], unattached volumes=[hooks mongodb-replica-set-keyfile tls-secret data-volume healthstatus tls-ca kube-api-access-45qcl agent-scripts logs-volume automation-config]: timed out waiting for the condition

2. Fix the problematic MongoDB pods by running the following script:

   #!/bin/bash
   
   set -eu
   
   
   FAILED_PVC_LIST=""
   STORAGE_CLASS_SINGLE_REPLICA="longhorn-backup-single-replica"
   STORAGE_CLASS="longhorn-backup"
   LOCAL_RESTORE_PATH="restoredata"
   mkdir -p ${LOCAL_RESTORE_PATH}
   export LOCAL_RESTORE_PATH="${LOCAL_RESTORE_PATH}"
   
   
   function delete_pvc_resources(){
     local pv_name=$1
     local volumeattachment_name=$2
   
     echo "deleting pv & volumes forcefully"
     delete_pv_forcefully "${pv_name}"
     sleep 2
     delete_longhornvolume_forcefully "${pv_name}"
     sleep 2
     if [ -n "${volumeattachment_name}" ]; then
       echo "deleting volumeattachments forcefully"
       delete_volumeattachment_forcefully "${volumeattachment_name}"
       sleep 5
     fi
   }
   
   function delete_longhornvolume_forcefully() {
     local pv_name=$1
     
     if ! kubectl -n longhorn-system get volumes.longhorn.io "${pv_name}" >/dev/null 2>&1 ; then
       echo "Volume ${pv_name} not found, skip deletion."
       return
     fi
   
     kubectl -n longhorn-system delete volumes.longhorn.io "${pv_name}" --grace-period=0 --force &
     local success=0
     local try=0
     local maxtry=10
     while (( try < maxtry )); do
       local result=""
       # update finaluzer field to null
       result=$(kubectl -n longhorn-system get volumes.longhorn.io "${pv_name}" -o=json | jq '.metadata.finalizers = null' | kubectl apply -f -) || true
       echo "${result}"
   
       result=$(kubectl -n longhorn-system get volumes.longhorn.io "${pv_name}")|| true
       echo "${result}"
       if [[ -z "${result}" ]]; then
         success=1
         break;
       fi
       kubectl -n longhorn-system delete volumes.longhorn.io "${pv_name}" --grace-period=0 --force &
       echo "Waiting to delete volume ${pv_name} ${try}/${maxtry}..."; sleep 10
       try=$(( try + 1 ))
     done
   
      if [ "${success}" -eq 0 ]; then
       echo "Failed to delete volume ${pv_name}."
     fi
   
   }
   
   function delete_pv_forcefully() {
     local pv_name=$1
   
     kubectl delete pv "${pv_name}" --grace-period=0 --force &
   
     local success=0
     local try=0
     local maxtry=10
     while (( try < maxtry )); do
       # update finaluzer field to null
       result=$(kubectl get pv "${pv_name}" -o=json | jq '.metadata.finalizers = null' | kubectl apply -f -) || true
       echo "${result}"
   
       result=$(kubectl get pv "${pv_name}")|| true
       echo "${result}"
       if [[ -z "${result}" ]]; then
         success=1
         break;
       fi
       kubectl delete pv "${pv_name}" --grace-period=0 --force &
       echo "Waiting to delete pv ${pv_name} ${try}/${maxtry}..."; sleep 10
       try=$(( try + 1 ))
     done
   
     if [ "${success}" -eq 0 ]; then
       echo "Failed to delete pv ${pv_name}."
     fi
   }
   
   function validate_pv_backup_available(){
     local pv_name=$1
     local validate_pv_backup_available_resp=0
     local backup_resp=""
     local resp_status_code=""
     local resp_message=""
   
     local try=0
     local maxtry=3
     while (( try != maxtry )) ; do
       backup_resp=$(curl --noproxy "*" "${LONGHORN_URL}/v1/backupvolumes/${pv_name}?") || true
       echo "Backup Response: ${backup_resp}"
       if { [ -n "${backup_resp}" ] && [ "${backup_resp}" != " " ]; }; then
         resp_status_code=$(echo "${backup_resp}"| jq -c ".status")
         resp_message=$(echo "${backup_resp}"| jq -c ".message")
         if [[ -n "${resp_status_code}" && "${resp_status_code}" != " " && "${resp_status_code}" != "null" && (( resp_status_code -gt 200 )) ]] ; then
           validate_pv_backup_available_resp=0
         else
           resp_message=$(echo "${backup_resp}"| jq -c ".messages.error")
           if { [ -z "${resp_message}" ] || [ "${resp_message}" = "null" ]; }; then
             echo "PVC Backup is available for ${pv_name}"
             # shellcheck disable=SC2034
             validate_pv_backup_available_resp=1
             break;
           fi
         fi
       fi
       try=$((try+1))
       sleep 10
     done
     export IS_BACKUP_AVAILABLE="${validate_pv_backup_available_resp}"
   }
   
   function store_pvc_resources(){
     local pvc_name=$1
     local namespace=$2
     local pv_name=$3
     local volumeattachment_name=$4
   
     if [[ -n "${volumeattachment_name}" && "${volumeattachment_name}" != " " ]]; then
       result=$(kubectl get volumeattachments "${volumeattachment_name}" -o json > "${LOCAL_RESTORE_PATH}/${volumeattachment_name}".json) || true
       echo "${result}"
     fi
   
     kubectl get pv "${pv_name}" -o json > "${LOCAL_RESTORE_PATH}/${pv_name}"-pv.json
     kubectl -n longhorn-system get volumes.longhorn.io "${pv_name}" -o json > "${LOCAL_RESTORE_PATH}/${pv_name}"-volume.json
     kubectl get pvc "${pvc_name}" -n "${namespace}" -o json > "${LOCAL_RESTORE_PATH}/${pvc_name}"-pvc.json
   }
   
   function wait_pvc_bound() {
     local namespace=$1
     local pvc_name=$2
   
     try=0
     maxtry=30
     while (( try < maxtry )); do
       local status=""
       status=$(kubectl -n "${namespace}" get pvc "${pvc_name}" -o json| jq -r ".status | select( has(\"phase\")).phase")
       if [[ "${status}"  == "Bound" ]]; then
         echo "PVC ${pvc_name} Bouned successfully."
         break;
       fi
       echo "waiting for PVC to bound...${try}/${maxtry}"; sleep 30
       try=$((try+1))
     done
   }
   
   function create_volume() {
     local pv_name=$1
     local backup_path=$2
     local response
     create_volume_status="succeed"
     echo "Creating Volume with PV: ${pv_name} and BackupPath: ${backup_path}"
     local accessmode
     local replicacount
     accessmode=$(< "${LOCAL_RESTORE_PATH}/${pv_name}-volume.json" jq -r ".spec.accessMode")
     replicacount=$(< "${LOCAL_RESTORE_PATH}/${pv_name}-volume.json" jq -r ".spec.numberOfReplicas")
   
     # create volume from backup pv
     response=$(curl --noproxy "*" "${LONGHORN_URL}/v1/volumes" -H 'Accept: application/json' -H 'Content-Type: application/json;charset=UTF-8' --data-raw "{\"name\":\"${pv_name}\", \"accessMode\":\"${accessmode}\", \"fromBackup\": \"${backup_path}\", \"numberOfReplicas\": ${replicacount}}")
   
     sleep 5
   
     if [[ -n "${response}" && "${response}" != "null" && "${response}" != " " && -n $(echo "${response}"|jq -r ".id") ]]; then
     # wait for volume to be detached , max wait 4hr
       local try=0
       local maxtry=480
       local success=0
       while (( try < maxtry ));do
         status=$(kubectl -n longhorn-system get volumes.longhorn.io "${pv_name}"  -o json |jq -r ".status.state") || true
         echo "volume ${pv_name} status: ${status}"
         if [[ "${status}" == "detached" ]]; then
           # update label
           kubectl -n longhorn-system label volumes.longhorn.io/"${pv_name}" recurring-job-group.longhorn.io/uipath-backup=enabled
           success=1
           echo "Volume ready to use"
           break
         fi
         echo "waiting for volume to be ready to use${try}/${maxtry}..."; sleep 30
         restore_status_url="${LONGHORN_URL}/v1/volumes/${pv_name}?action=snapshotList"
         try=$(( try + 1 ))
       done
   
       if [ "${success}" -eq 0 ]; then
         create_volume_status="failed"
         echo "${pv_name} Volume is not ready to use with status ${status}"
       fi
     else
       create_volume_status="failed"
       kubectl create -f "${LOCAL_RESTORE_PATH}/${pv_name}"-volume.json || true
       echo "${pv_name} Volume creation failed ${response} "
     fi
     echo "${create_volume_status}"
   }
   
   function restore_with_backupvolume() {
     local pvc_name=$1
     local namespace=$2
     local pv_name=$3
     local volumeattachment_name=$4
     local backup_path=$5
     create_volume_status="succeed"
     store_pvc_resources "${pvc_name}" "${namespace}" "${pv_name}" "${volumeattachment_name}"
     sleep 5
     delete_pvc_resources "${pv_name}" "${volumeattachment_name}"
     sleep 5
     create_volume "${pv_name}" "${backup_path}"
     sleep 10
   
     kubectl create -f "${LOCAL_RESTORE_PATH}/${pv_name}"-pv.json
     if [[ -n "${create_volume_status}" && "${create_volume_status}" != "succeed" ]]; then
       echo "Backup volume restore failed for pvc ${pvc_name} in namespace ${namespace}"
       restore_pvc_status="failed"
     else 
       local pvc_uid=""
       pvc_uid=$(kubectl -n "${namespace}" get pvc "${pvc_name}" -o json| jq -r ".metadata.uid")
   
       # update pv with pvc uid
       kubectl patch pv "${pv_name}" --type json -p "[{\"op\": \"add\", \"path\": \"/spec/claimRef/uid\", \"value\": \"${pvc_uid}\"}]"
   
       wait_pvc_bound "${namespace}" "${pvc_name}"
       echo "${result}"
     fi
     echo ${restore_pvc_status}
   }
   
   function restore_pvc(){
     local pvc_name=$1
     local namespace=$2
     local pv_name=$3
     local volumeattachment_name=$4
     local backup_path=$5
     restore_pvc_status="succeed"
     restore_with_backupvolume "${pvc_name}" "${namespace}" "${pv_name}" "${volumeattachment_name}" "${backup_path}"
     echo ${restore_pvc_status}
     # attach_volume "${pv_name}"
   }
   
   function get_backup_list_by_pvname(){
     local pv_name=$1
     local get_backup_list_by_pvname_resp=""
     local backup_list_response=""
   
     local try=0
     local maxtry=3
     while (( try != maxtry )) ; do
   
       backup_list_response=$(curl --noproxy "*" "${LONGHORN_URL}/v1/backupvolumes/${pv_name}?action=backupList" -X 'POST' -H 'Content-Length: 0' -H 'Accept: application/json')
       if [[ -n "${backup_list_response}" && "${backup_list_response}" != " " && -n $( echo "${backup_list_response}"|jq ".data" )  && -n $( echo "${backup_list_response}"|jq ".data[]" ) ]]; then
         echo "Backup List Response: ${backup_list_response}"
         # pick first backup data with url non empty
         # shellcheck disable=SC2034
         get_backup_list_by_pvname_resp=$(echo "${backup_list_response}"|jq -c '[.data[]|select(.url | . != null and . != "")][0]')
         break;
       fi
       try=$((try+1))
       sleep 10
     done
     export PV_BACKUP_PATH="${get_backup_list_by_pvname_resp}"
   }
   
   function get_pvc_resources() {
   
     get_pvc_resources_resp=""
   
     local PVC_NAME=$1
     local PVC_NAMESPACE=$2
   
     # PV have one to one mapping with PVC
     PV_NAME=$(kubectl -n "${PVC_NAMESPACE}" get pvc "${PVC_NAME}" -o json|jq -r ".spec.volumeName")
   
     VOLUME_ATTACHMENT_LIST=$(kubectl get volumeattachments -n "${PVC_NAMESPACE}" -o=json|jq -c ".items[]|\
   {name: .metadata.name, pvClaimName:.spec.source| select( has (\"persistentVolumeName\")).persistentVolumeName}")
   
     VOLUME_ATTACHMENT_NAME=""
     for VOLUME_ATTACHMENT in ${VOLUME_ATTACHMENT_LIST};
     do
       PV_CLAIM_NAME=$(echo "${VOLUME_ATTACHMENT}"|jq -r ".pvClaimName")
       if [[ "${PV_NAME}" = "${PV_CLAIM_NAME}" ]]; then
         VOLUME_ATTACHMENT_NAME=$(echo "${VOLUME_ATTACHMENT}"|jq -r ".name")
         break;
       fi
     done
   
     BACKUP_PATH=""
     validate_pv_backup_available "${PV_NAME}"
     local is_backup_available=${IS_BACKUP_AVAILABLE:- 0}
     unset IS_BACKUP_AVAILABLE
   
     if [ "${is_backup_available}" -eq 1 ]; then
       echo "Backup is available for PVC ${PVC_NAME}"
   
       get_backup_list_by_pvname "${PV_NAME}"
       BACKUP_PATH=$(echo "${PV_BACKUP_PATH}"| jq -r ".url")
       unset PV_BACKUP_PATH
     fi
   
     get_pvc_resources_resp="{\"pvc_name\": \"${PVC_NAME}\", \"pv_name\": \"${PV_NAME}\", \"volumeattachment_name\": \"${VOLUME_ATTACHMENT_NAME}\", \"backup_path\": \"${BACKUP_PATH}\"}"
     echo "${get_pvc_resources_resp}"
   }
   
   function scale_ownerreferences() {
     local ownerReferences=$1
     local namespace=$2
     local replicas=$3
   
     # no operation required
     if [[ -z "${ownerReferences}" || "${ownerReferences}" == "null" ]]; then
       return
     fi
   
     ownerReferences=$(echo "${ownerReferences}"| jq -c ".[]")
     for ownerReference in ${ownerReferences};
     do
       echo "Owner: ${ownerReference}"
       local resourceKind
       local resourceName
       resourceKind=$(echo "${ownerReference}"| jq -r ".kind")
       resourceName=$(echo "${ownerReference}"| jq -r ".name")
   
       if kubectl -n "${namespace}" get "${resourceKind}" "${resourceName}" >/dev/null 2>&1; then
         # scale replicas
         kubectl  -n "${namespace}" patch "${resourceKind}" "${resourceName}" --type json -p "[{\"op\": \"replace\", \"path\": \"/spec/members\", \"value\": ${replicas} }]"
       fi
     done
   }
   
   function scale_down_statefulset() {
     local statefulset_name=$1
     local namespace=$2
     local ownerReferences=$3
   
     echo "Start Scale Down statefulset ${statefulset_name} under namespace ${namespace}..."
   
     # validate and scale down ownerreference
     scale_ownerreferences "${ownerReferences}" "${namespace}" 0
   
     local try=0
     local maxtry=30
     success=0
     while (( try != maxtry )) ; do
       result=$(kubectl scale statefulset "${statefulset_name}" --replicas=0 -n "${namespace}") || true
       echo "${result}"
       scaledown=$(kubectl get statefulset "${statefulset_name}" -n "${namespace}"|grep 0/0) || true
       if { [ -n "${scaledown}" ] && [ "${scaledown}" != " " ]; }; then
         echo "Statefulset scaled down successfully."
         success=1
         break
       else
         try=$((try+1))
         echo "waiting for the statefulset ${statefulset_name} to scale down...${try}/${maxtry}";sleep 30
       fi
     done
   
     if [ ${success} -eq 0 ]; then
       echo "Statefulset ${statefulset_name} scaled down failed"
     fi
   }
   
   function scale_up_statefulset() {
     local statefulset_name=$1
     local namespace=$2
     local replica=$3
     local ownerReferences=$4
   
     # Scale up statefulsets using PVCs
     echo "Start Scale Up statefulset ${statefulset_name}..."
   
     # validate and scale up ownerreference
     scale_ownerreferences "${ownerReferences}" "${namespace}" "${replica}"
   
     echo "Waiting to scale up statefulset..."
   
     local try=1
     local maxtry=15
     local success=0
     while (( try != maxtry )) ; do
   
       kubectl scale statefulset "${statefulset_name}" --replicas="${replica}" -n "${namespace}"
       kubectl get statefulset "${statefulset_name}" -n "${namespace}"
   
       scaleup=$(kubectl get statefulset "${statefulset_name}" -n "${namespace}"|grep "${replica}"/"${replica}") || true
       if ! { [ -n "${scaleup}" ] && [ "${scaleup}" != " " ]; }; then
         try=$((try+1))
         echo "waiting for the statefulset ${statefulset_name} to scale up...${try}/${maxtry}"; sleep 30
       else
         echo "Statefulset scaled up successfully."
         success=1
         break
       fi
     done
   
     if [ ${success} -eq 0 ]; then
       echo "Statefulset scaled up failed ${statefulset_name}."
     fi
   }
   
   function restore_pvc_attached_to_statefulsets() {
     local namespace
     namespace="${1}"
     local statefulset_list
   
     # list of all statefulset using PVC
     statefulset_list=$(kubectl get statefulset -n "${namespace}" -o=json | jq -r ".items[] | select(.spec.volumeClaimTemplates).metadata.name")
   
     for statefulset_name in ${statefulset_list};
     do
       local replica
       local ownerReferences
       local pvc_restore_failed
       local try=0
       local maxtry=5
       local status="notready"
       pvc_restore_failed=""
       restore_pvc_status=""
   
       # check if statefulset is reday
       while [[ "${status}" == "notready" ]] && (( try < maxtry )); do
         echo "fetch statefulset ${statefulset_name} metadata...  ${try}/${maxtry}"
         try=$(( try + 1 ))
         replica=$(kubectl get statefulset "${statefulset_name}" -n "${namespace}" -o=json | jq -c ".spec.replicas")
   
         if [[ "${replica}" != 0 ]]; then
           status="ready"
         else
           echo "statefulset ${statefulset_name} replica is not ready. Wait and retry"; sleep 30
         fi
       done
   
       if [[ "${status}" != "ready" ]]; then
         echo "Failed to restore pvc for Statefulset ${statefulset_name} in namespace ${namespace}. Please retrigger volume restore step."
       fi
   
       # Fetch ownerReferences and claim name
       ownerReferences=$(kubectl get statefulset "${statefulset_name}" -n "${namespace}" -o=json | jq -c ".metadata.ownerReferences")
       claimTemplatesName=$(kubectl get statefulset "${statefulset_name}" -n "${namespace}" -o=json | jq -c ".spec | select( has (\"volumeClaimTemplates\") ).volumeClaimTemplates[].metadata.name " | xargs)
       
       echo "Scaling down Statefulset ${statefulset_name} with ${replica} under namespace ${namespace}"
       scale_down_statefulset "${statefulset_name}" "${namespace}" "${ownerReferences}"
       for name in ${claimTemplatesName}; do
         local pvc_prefix
         pvc_prefix="${name}-${statefulset_name}"
   
         for((i=0;i<"${replica}";i++)); do
           local pvc_name
           pvc_name="${pvc_prefix}-${i}"
   
           pvc_exist=$(kubectl -n "${namespace}" get pvc "${pvc_name}") || true
           if [[ -z "${pvc_exist}" || "${pvc_exist}" == " " ]]; then
             echo "PVC not available for the statefulset ${statefulset_name}, skipping restore."
             continue;
           fi
   
           local pvc_storageclass
           pvc_storageclass=$(kubectl -n "${namespace}" get pvc "${pvc_name}" -o json| jq -r ".spec.storageClassName")
           if [[ ! ( "${pvc_storageclass}" == "${STORAGE_CLASS}" || "${pvc_storageclass}" == "${STORAGE_CLASS_SINGLE_REPLICA}" ) ]]; then
             echo "backup not available for pvc ${pvc_name}, storageclass: ${pvc_storageclass} "
             continue;
           fi
   
           # get pv, volumeattachments for pvc
           get_pvc_resources_resp=""
           get_pvc_resources "${pvc_name}" "${namespace}"
   
           local pv_name
           local volumeattachment_name
           local backup_path
           pv_name=$(echo "${get_pvc_resources_resp}"| jq -r ".pv_name")
           volumeattachment_name=$(echo "${get_pvc_resources_resp}"| jq -r ".volumeattachment_name")
           backup_path=$(echo "${get_pvc_resources_resp}"| jq -r ".backup_path")
           if [[ -z "${backup_path}" || "${backup_path}" == " " || "${backup_path}" == "null" ]]; then
             pvc_restore_failed="error"
             FAILED_PVC_LIST="${FAILED_PVC_LIST},${pv_name}"
             continue;
           fi
   
           restore_pvc_status="succeed"
           restore_pvc "${pvc_name}" "${namespace}" "${pv_name}" "${volumeattachment_name}" "${backup_path}"
           if [[ -n "${restore_pvc_status}" && "${restore_pvc_status}" != "succeed" ]]; then
             pvc_restore_failed="error"
             FAILED_PVC_LIST="${FAILED_PVC_LIST},${pv_name}"
             continue;
           fi
         done
       done
   
       sleep 10
       scale_up_statefulset "${statefulset_name}" "${namespace}" "${replica}" "${ownerReferences}"
       sleep 5
   
       if [[ -n "${pvc_restore_failed}" && "${pvc_restore_failed}" == "error" ]]; then
           echo "Failed to restore pvc for Statefulset ${statefulset_name} in namespace ${namespace}."
       fi    
     done
   }
   
   LONGHORN_URL=$(kubectl -n longhorn-system get svc longhorn-backend -o jsonpath="{.spec.clusterIP}"):9500
   
   restore_pvc_attached_to_statefulsets "mongodb"