From a67d0cf3f3055e2cdb99d9f46a97f2d7c06df540 Mon Sep 17 00:00:00 2001
From: Joseph Spiker <jspiker@carepayment.com>
Date: Wed, 15 Jul 2026 16:57:25 +0000
Subject: [PATCH] fix(bao-secrets): retry login+fetch with backoff on
 429/504/non-JSON (#545/#590-adjacent)

Under a CI burst (a batch merge fires ~13 workflows x ~4 jobs, each doing an
AppRole login = a raft write), the vault nodes return nginx 504s and the
rate-limit quota returns 429s. Both are non-JSON, which crashed jq and
hard-failed the job with a misleading "parse error". Add a bao_req helper that
retries connect-failures/429/5xx with backoff+jitter (~60s over 6 tries) and
surfaces real 4xx bodies unchanged. Transient throttling no longer fails CI.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 bao-secrets/action.yml | 51 ++++++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/bao-secrets/action.yml b/bao-secrets/action.yml
index 885d84a..f639c72 100644
--- a/bao-secrets/action.yml
+++ b/bao-secrets/action.yml
@@ -35,14 +35,42 @@ runs:
         command -v curl >/dev/null || { echo "::error::bao-secrets needs 'curl' on the runner"; exit 1; }
         command -v jq   >/dev/null || { echo "::error::bao-secrets needs 'jq' on the runner"; exit 1; }
 
-        # 1) AppRole login -> short-lived (15m) token. No -f so we can read the
-        #    error body; the token check below is the real gate.
-        login=$(curl -s --max-time 15 -X POST \
-          --data "{\"role_id\":\"${BAO_ROLE_ID}\",\"secret_id\":\"${BAO_SECRET_ID}\"}" \
-          "${BAO_ADDR}/v1/auth/approle/login") || true
-        TOKEN=$(printf '%s' "$login" | jq -r '.auth.client_token // empty')
+        TOKEN=""
+        # HTTP with retry + backoff. When many CI jobs log in at once the vault
+        # nodes can return an nginx 504 (login is a raft write) or a 429 from the
+        # rate-limit quota — both are transient and non-JSON, which used to crash
+        # jq and hard-fail the job. Retry those; surface real 4xx bodies as-is.
+        # Prints the response BODY on stdout.
+        bao_req() { # METHOD URL [DATA]
+          local method="$1" url="$2" data="${3:-}" attempt=1 max=6 code tmp secs
+          tmp="$(mktemp)"
+          local -a args=(-s -o "$tmp" -w '%{http_code}' --max-time 15 -X "$method")
+          [ -n "${TOKEN:-}" ] && args+=(-H "X-Vault-Token: ${TOKEN}")
+          [ -n "$data" ] && args+=(--data "$data")
+          args+=("$url")
+          while :; do
+            code=$(curl "${args[@]}" 2>/dev/null || echo 000)
+            case "$code" in
+              200|204) cat "$tmp"; rm -f "$tmp"; return 0 ;;
+              000|429|500|502|503|504)
+                if [ "$attempt" -ge "$max" ]; then
+                  echo "::warning::bao $method failed after ${attempt} tries (last HTTP ${code})" >&2
+                  cat "$tmp"; rm -f "$tmp"; return 1
+                fi
+                secs=$(( attempt * 4 + RANDOM % 4 ))   # ~4,8,12,16,20s + jitter
+                echo "  bao ${method} -> HTTP ${code}; retry ${attempt}/${max} in ${secs}s (cluster busy/rate-limited)" >&2
+                sleep "$secs"; attempt=$((attempt+1)) ;;
+              *) cat "$tmp"; rm -f "$tmp"; return 1 ;;   # 4xx etc — surface to caller
+            esac
+          done
+        }
+
+        # 1) AppRole login -> short-lived token (batch token if the role is set so).
+        login="$(bao_req POST "${BAO_ADDR}/v1/auth/approle/login" \
+          "{\"role_id\":\"${BAO_ROLE_ID}\",\"secret_id\":\"${BAO_SECRET_ID}\"}")" || true
+        TOKEN=$(printf '%s' "$login" | jq -r '.auth.client_token // empty' 2>/dev/null || true)
         if [ -z "$TOKEN" ]; then
-          echo "::error::OpenBao AppRole login failed: $(printf '%s' "$login" | jq -rc '.errors // "no response / network error"')"
+          echo "::error::OpenBao AppRole login failed (after retries): $(printf '%s' "$login" | jq -rc '.errors // "non-JSON/timeout — cluster busy or rate-limited"' 2>/dev/null || echo 'non-JSON response')"
           exit 1
         fi
         echo "::add-mask::$TOKEN"
@@ -59,11 +87,10 @@ runs:
           fi
           env_name="$1"; path="$2"; field="$3"
           mount="${path%%/*}"; rest="${path#*/}"      # KV v2: <mount>/data/<rest>
-          resp=$(curl -s --max-time 15 -H "X-Vault-Token: $TOKEN" \
-            "${BAO_ADDR}/v1/${mount}/data/${rest}") || true
-          val=$(printf '%s' "$resp" | jq -r --arg f "$field" '.data.data[$f] // empty')
+          resp="$(bao_req GET "${BAO_ADDR}/v1/${mount}/data/${rest}")" || true
+          val=$(printf '%s' "$resp" | jq -r --arg f "$field" '.data.data[$f] // empty' 2>/dev/null || true)
           if [ -z "$val" ]; then
-            echo "::error::no value at '${path}' field '${field}' (wrong path/field, or this role's policy denies it): $(printf '%s' "$resp" | jq -rc '.errors // "empty"')"
+            echo "::error::no value at '${path}' field '${field}' (wrong path/field, this role's policy denies it, or cluster busy): $(printf '%s' "$resp" | jq -rc '.errors // "empty/non-JSON"' 2>/dev/null || echo 'non-JSON')"
             exit 1
           fi
           # mask every line of the value (handles multiline secrets like keys)
@@ -76,6 +103,6 @@ runs:
           echo "  ✓ ${env_name}  <-  ${path}#${field}"
         done <<< "$BAO_SECRETS"
 
-        # 3) drop the token (short-lived anyway — just tidy)
+        # 3) drop the token (batch tokens can't be revoked; ignore errors)
         curl -s --max-time 10 -H "X-Vault-Token: $TOKEN" \
           -X POST "${BAO_ADDR}/v1/auth/token/revoke-self" >/dev/null 2>&1 || true