Échec de la compression des métriques en raison de blocs corrompus dans Thanos

Réparez les blocs de mesure corrompus dans le compacteur Thanos pour la surveillance Automation Suite sous Linux.

Description

Le compacteur Thanos peut ne pas compacter les métriques lorsque des blocs corrompus sont détectés dans le magasin d’objets. Cette condition empêche le compacteur de traiter les métriques, ce qui entraîne une utilisation accrue du stockage dans le compartiment Ceph.

Solution

Pour résoudre ce problème, procédez comme suit :

1. Sur n'importe quel nœud de serveur, exécutez le script suivant :

   thanosns=monitoring && if kubectl get application -n argocd rancher-monitoring; then thanosns=cattle-monitoring-system; fi && cat <<EOF | kubectl apply -f -
   ---
   apiVersion: rbac.authorization.k8s.io/v1
   kind: Role
   metadata:
     annotations:
     labels:
       app.kubernetes.io/component: thanos-cleaner
       app.kubernetes.io/instance: thanos-block-cleaner
       app.kubernetes.io/name: thanos-block-cleaner
     name: thanos-cleaner-role
     namespace: ${thanosns}
   rules:
   - apiGroups:
     - apps
     resources:
     - statefulsets
     - statefulsets/scale
     verbs:
     - list
     - get
     - update
     - patch
   - apiGroups:
     - batch
     resources:
     - jobs
     - cronjobs
     verbs:
     - delete
     - list
     - get
     - update
     - create
     - watch
   - apiGroups:
     - ""
     resources:
     - pods
     verbs:
     - delete
     - list
     - get
     - update
   ---
   apiVersion: rbac.authorization.k8s.io/v1
   kind: RoleBinding
   metadata:
     labels:
       app.kubernetes.io/component: thanos-cleaner
       app.kubernetes.io/instance: thanos-block-cleaner
       app.kubernetes.io/name: thanos-block-cleaner
     name: thanos-cleaner-role-binding
     namespace: ${thanosns}
   roleRef:
     apiGroup: rbac.authorization.k8s.io
     kind: Role
     name: thanos-cleaner-role
   subjects:
   - kind: ServiceAccount
     name: thanos-cleaner
     namespace: ${thanosns}
   ---
   apiVersion: v1
   kind: ServiceAccount
   metadata:
     name: thanos-cleaner
     namespace: ${thanosns}
   ---
   apiVersion: monitoring.coreos.com/v1
   kind: PrometheusRule
   metadata:
     name: thanos-cleaner
     namespace: uipath
   spec:
     groups:
     - name: thanos
       rules:
       - alert: ThanosCompactorNotWorking
         annotations:
           description: Thanos compactor is not working. This will disable metrics compaction
             in objectstore bucket. Please check thanos compact pod in ${thanosns} namespace
             for any error. Compactor in faulty state will exhaust object store space
           message: Thanos compactor is not working. Please check if thanos cleaner job
             is functional and able to fix corruption
           runbook_url: https://docs.uipath.com/automation-suite/docs/alert-runbooks
           summary: Thanos compactor is not working
         expr: thanos_compactor_issue{job="thanos-cleaner"} >= 1
         for: 1d
         labels:
           app: thanos
           severity: critical
   ---
   EOF
   

2. Sur n'importe quel nœud de serveur, exécutez le script suivant :

   cat <<'EOF' | kubectl apply -f -
   ---
   apiVersion: v1
   data:
     thanos-cleanup.sh: |
       #!/bin/bash
   
       # Copyright UiPath 2021
       #
       # =================
       # LICENSE AGREEMENT
       # -----------------
       #   Use of paid UiPath products and services is subject to the licensing agreement
       #   executed between you and UiPath. Unless otherwise indicated by UiPath, use of free
       #   UiPath products is subject to the associated licensing agreement available here:
       #   https://www.uipath.com/legal/trust-and-security/legal-terms (or successor website).
       #   You must not use this file separately from the product it is a part of or is associated with.
   
       set -eu -o pipefail
   
       export PATH=$PATH:/thanos-bin/
       # Below script removes the blocks which are overlapping or having index issue or having duplicated compaction
       #
       # In few cases with above mentioned scenarios, thanos may skip the compaction and halt the compaction module.
       # Compaction halt requires manual deletion of corrupted blocks and restart of compact pod.
   
       config_file=/etc/thanos/${THANOS_CONFIG_KEY}
   
       function info() {
         echo "[INFO] [$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*"
       }
   
       function warn() {
         echo -e "\e[0;33m[WARN] [$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
       }
   
       function error_without_exit() {
         echo -e "\e[0;31m[ERROR][$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
       }
   
       function error() {
         echo -e "\e[0;31m[ERROR][$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
         exit 1
       }
   
       function is_compaction_halted() {
         info "Checking if thanos compactor running"
   
         IFS=" " read -r -a compactor_addresses <<<"$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=thanos-compact -o jsonpath="{.items[*].status.podIP}")"
   
         is_compactor_halted=0
   
         if [[ "${#compactor_addresses[@]}" -eq 0 ]]; then
           info "Thanos compactor pod is not running"
           is_compactor_halted=1
         fi
   
         for ip in "${compactor_addresses[@]}"; do
           #shellcheck disable=SC2086
           halted=$(curl -s http://${ip}:10902/metrics | grep thanos_compact_halted | grep -v '#' | awk -F ' ' '{print $2}')
           if [[ "$halted" -eq "1" ]]; then
             warn "Compaction is halted"
             is_compactor_halted=1
             break
           fi
         done
   
         return $is_compactor_halted
       }
   
       function execute_thanos_issue_command() {
         if [[ $# -ne 1 ]]; then
           error "missing issue name for execute_thanos_issue_command function"
         fi
   
         issue=$1
   
         info "Checking for issue $issue"
         cmd_ret=0
         #shellcheck disable=SC2086
         verify_output=$(thanos tools bucket --objstore.config-file=${config_file} verify --log.format=json -i $issue 2>&1) && true || cmd_ret=1
         if [[ $cmd_ret -eq 1 ]]; then
           error_without_exit "Output of $issue command: -> $verify_output"
           error "Failed to verify bucket for $issue"
         fi
   
         #shellcheck disable=SC2086
         echo $verify_output
       }
   
       function fix_index_issue() {
         info "Fixing index_known_issue issue"
   
         verify_output=$(execute_thanos_issue_command "index_known_issues")
         #shellcheck disable=SC2086
         for b in $(echo $verify_output | sed 's/} {/\r\n/g' | grep err | grep "detected issue" | awk -F '"id":' '{print $2}' | awk -F ',' '{print $1}' | tr -d '"'); do
           info "Block=$b is having the issue, removing it.."
   
           thanos tools bucket mark --id="$b" \
             --marker=deletion-mark.json \
             --details="deleted by job" \
             --objstore.config-file="${config_file}"
   
           info "Block=$b is marked for deletion"
         done
   
         info "Fixing index_known_issue issue done"
       }
   
       function fix_overlapping_issue() {
         info "Fixing overlapped_blocks issue"
   
         overlap_output=$(execute_thanos_issue_command "overlapped_blocks")
   
         while IFS= read -r line; do
           #shellcheck disable=SC2086
           for b in $(echo $line | awk -F '"overlap":' '{print $2}' | awk -v search="ulid" 'match($0, search) {print substr($0, RSTART)}' | sed 's/ulid/\r\nulid/g' | awk -F ',' '{print $1}' | grep '^ulid' | awk -F ': ' '{print $2}'); do
             info "Block=$b is having the issue, removing it.."
             thanos tools bucket mark --id="$b" \
               --marker=deletion-mark.json \
               --details="deleted by job" \
               --objstore.config-file="${config_file}"
   
             info "Block=$b is marked for deletion"
           done
         done < <(echo "$overlap_output" | sed 's/} {/\r\n/g' | grep "found overlapped blocks")
   
         info "Fixing overlapped_blocks issue done"
       }
   
       function fix_duplicate_issue() {
         info "Fixing duplicated_compaction issue"
         duplicate_output=$(execute_thanos_issue_command "duplicated_compaction")
         #shellcheck disable=SC2086,SC2006
         for b in $(echo $duplicate_output | sed 's/ts=2/\r\n2/g' | grep "Found duplicated blocks that are ok to be removed" | awk -F 'ULIDs="' '{print $2}' | tr -d '[]' | awk -F '"' '{print $1}'); do
           info "Block=$b is having the issue, removing it.."
           thanos tools bucket mark --id="$b" \
             --marker=deletion-mark.json \
             --details="deleted by job" \
             --objstore.config-file="${config_file}"
   
           info "Block=$b is marked for deletion"
         done
   
         info "Fixing duplicated_compaction issue done"
       }
   
       if [[ -z "$NAMESPACE" ]]; then
         error "NAMESPACE is not set"
       fi
   
       # We will check if compaction is halted or not before checking for issues
       if is_compaction_halted; then
         info "Thanos compaction is working"
         echo "thanos_compactor_issue 0" | curl --data-binary @- "http://pushgateway-prometheus-pushgateway.uipath.svc.cluster.local:9091/metrics/job/thanos-cleaner"
         exit 0
       fi
   
       warn "Thanos compactor is not working. Checking for corrupted blocks..."
       echo "thanos_compactor_issue 1" | curl --data-binary @- "http://pushgateway-prometheus-pushgateway.uipath.svc.cluster.local:9091/metrics/job/thanos-cleaner"
   
       if [[ "$DISABLE_BLOCK_CLEANER" == true ]]; then
         info "DISABLE_BLOCK_CLEANER is set to $DISABLE_BLOCK_CLEANER, skipping block clean"
         exit 0
       fi
   
       info "DISABLE_BLOCK_CLEANER is set to $DISABLE_BLOCK_CLEANER, removing corrupted blocks"
   
       replica=$(kubectl get sts -n "$NAMESPACE" thanos-compact -o jsonpath='{.spec.replicas}')
   
       # compactor must not be running while deleting blocks
   
       info "Stopping compactor"
       kubectl scale sts -n "$NAMESPACE" thanos-compact --replicas=0
       kubectl delete pods -n "$NAMESPACE" -l app.kubernetes.io/instance=thanos-compact --force
   
       # fixing index_known_issues
       info "Checking blocks having issue"
   
       fix_index_issue
       fix_overlapping_issue
       fix_duplicate_issue
   
       info "Triggering deletion of all marked blocks"
   
       #shellcheck disable=SC2086
       thanos tools bucket cleanup --delete-delay=0 --objstore.config-file=${config_file}
   
       info "Corrupted blocks are deleted"
   
       info "Scaling thanos compactor's replica to $replica"
       #shellcheck disable=SC2086
       kubectl scale sts -n "$NAMESPACE" thanos-compact --replicas=$replica
       info "Thanos compactor started"
     validate-cronjob.sh: |
       #!/bin/bash
   
       # Copyright UiPath 2021
       #
       # =================
       # LICENSE AGREEMENT
       # -----------------
       #   Use of paid UiPath products and services is subject to the licensing agreement
       #   executed between you and UiPath. Unless otherwise indicated by UiPath, use of free
       #   UiPath products is subject to the associated licensing agreement available here:
       #   https://www.uipath.com/legal/trust-and-security/legal-terms (or successor website).
       #   You must not use this file separately from the product it is a part of or is associated with.
   
       set -eu -o pipefail
   
       function info() {
         echo "[INFO] [$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*"
       }
   
       function warn() {
         echo -e "\e[0;33m[WARN] [$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
       }
   
       function error_without_exit() {
         echo -e "\e[0;31m[ERROR][$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
       }
   
       function error() {
         echo -e "\e[0;31m[ERROR][$(date +'%Y-%m-%dT%H:%M:%S%z')]:\e[0m $*" >&2
         exit 1
       }
   
       alias kubectl='kubectl --cache-dir=/tmp/'
       IFS="," read -ra cronjobs <<<"$CRONJOB_LIST"
   
       for cr in "${cronjobs[@]}"; do
         #shellcheck disable=SC2206
         name=(${cr//// })
         cronNs=default
         cronName=""
   
         if [[ ${#name[@]} -gt 2 || ${#name[@]} -lt 1 ]]; then
           error "Invalid cronjob name=$cr"
         fi
   
         if [[ ${#name[@]} -eq 2 ]]; then
           cronNs=${name[0]}
           cronName=${name[1]}
         else
           cronName=${name[0]}
         fi
   
         info "Validating cronjob=$cr"
   
         jobName="${cronName}-sf-job-validation"
   
         created=1
         info "Creating validation job for $cr"
         kubectl delete job -n "${cronNs}" "${jobName}" --ignore-not-found --timeout=3m
   
         #shellcheck disable=SC2086
         kubectl create job -n "${cronNs}" --from=cronjob/${cronName} "$jobName" || created=0
   
         if [[ $created == 0 ]]; then
           error "Failed to create job for $cr"
         fi
   
         #shellcheck disable=SC2086
         kubectl wait --timeout=20m --for=condition=complete -n "${cronNs}" job/$jobName &
         cpid=$!
   
         #shellcheck disable=SC2086
         kubectl wait --timeout=20m --for=condition=failed -n "${cronNs}" job/${jobName} && exit 1 &
         fpid=$!
   
         ret=0
         wait -n $cpid $fpid || ret=1
   
         kill -9 $cpid || true
         kill -9 $fpid || true
   
         if [[ $ret -eq 0 ]]; then
           info "Job for $cr is validated/completed"
           #ignore deletion error. if deletion fail then will get caught in next sync. This is to reduce failure during installation
           kubectl delete job -n "${cronNs}" "${jobName}" --timeout=3m || true
         else
           error "Job for $cr failed"
         fi
       done
   kind: ConfigMap
   metadata:
     name: thanos-cleaner-script
     namespace: monitoring
   ---
   EOF
   

3. Remplacez SF_K8S_TAG par la balise d'image correcte, puis appliquez l'élément cronjob.

   Depuis le répertoire du programme d'installation sur n'importe quel nœud de serveur, obtenez la dernière balise :

   cat versions/docker-images.json  |grep uipath/sf-k8-utils-rhel | tr -d ',"' | awk -F ':' '{print $2}' |sort |uniq |tail -1
   

   Mettez ensuite à jour le bloc cronjob en remplaçant SF_K8S_TAG par la valeur renvoyée.

   Une fois mis à jour, collez l'intégralité du bloc dans le terminal d'un nœud de serveur :

   thanosns=monitoring && if kubectl get application -n argocd rancher-monitoring; then thanosns=cattle-monitoring-system; fi && thanosimage=$(kubectl  get statefulset -n $thanosns thanos-compact -o jsonpath='{.spec.template.spec.containers[0].image}') &&  cat <<EOF | kubectl apply -f -
   ---
   apiVersion: batch/v1
   kind: CronJob
   metadata:
     name: thanos-cleaner
     namespace: ${thanosns}
   spec:
     concurrencyPolicy: Forbid
     failedJobsHistoryLimit: 3
     jobTemplate:
       metadata:
         creationTimestamp: null
       spec:
         backoffLimit: 3
         template:
           metadata:
             annotations:
               sidecar.istio.io/inject: "false"
             creationTimestamp: null
             labels:
               app.kubernetes.io/name: thanos-cleaner-cronjob
           spec:
             containers:
             - args:
               - /script/thanos-cleanup.sh
               command:
               - /bin/bash
               env:
               - name: NAMESPACE
                 valueFrom:
                   fieldRef:
                     apiVersion: v1
                     fieldPath: metadata.namespace
               - name: THANOS_CONFIG_KEY
                 value: thanos.yaml
               - name: DISABLE_BLOCK_CLEANER
                 value: "false"
               image: docker.io/uipath/sf-k8-utils-rhel:SF_K8S_TAG
               imagePullPolicy: IfNotPresent
               name: thanos-cleaner
               resources:
                 limits:
                   cpu: 200m
                   memory: 400Mi
                 requests:
                   cpu: 20m
                   memory: 64Mi
               terminationMessagePath: /dev/termination-log
               terminationMessagePolicy: File
               volumeMounts:
               - mountPath: /script/
                 name: script
               - mountPath: /etc/thanos/
                 name: thanos-objectstore-vol
               - mountPath: /thanos-bin/
                 name: thanos
               - mountPath: /.kube/
                 name: kubedir
               - mountPath: /tmp/
                 name: tmpdir
             dnsPolicy: ClusterFirst
             initContainers:
             - args:
               - set -e; cp /bin/thanos /thanos-bin/thanos && chmod +x /thanos-bin/thanos
               command:
               - /bin/sh
               - -c
               image: ${thanosimage}
               imagePullPolicy: IfNotPresent
               name: copy-uipathcore-binary
               resources: {}
               terminationMessagePath: /dev/termination-log
               terminationMessagePolicy: File
               volumeMounts:
               - mountPath: /thanos-bin/
                 name: thanos
             nodeSelector:
               kubernetes.io/os: linux
             restartPolicy: Never
             schedulerName: default-scheduler
             securityContext:
               fsGroup: 3000
               runAsGroup: 2000
               runAsNonRoot: true
               runAsUser: 1000
             serviceAccount: thanos-cleaner
             serviceAccountName: thanos-cleaner
             terminationGracePeriodSeconds: 120
             volumes:
             - emptyDir: {}
               name: kubedir
             - emptyDir: {}
               name: tmpdir
             - emptyDir: {}
               name: thanos
             - name: thanos-objectstore-vol
               secret:
                 defaultMode: 420
                 secretName: thanos-objectstore-config
             - configMap:
                 defaultMode: 420
                 name: thanos-cleaner-script
               name: script
     schedule: 0 1/6 * * *
     successfulJobsHistoryLimit: 2
     suspend: false
   ---
   EOF