333 lines
9.9 KiB
Bash
Executable File
333 lines
9.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
LOG_DIR="$ROOT_DIR/.test-logs"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
EXAMPLES=(
|
|
"hello-node"
|
|
"simple-node"
|
|
"simple-python"
|
|
"notes-python"
|
|
)
|
|
|
|
declare -A CHECK_METHOD=(
|
|
[hello-node]="POST"
|
|
[simple-node]="GET"
|
|
[simple-python]="GET"
|
|
[notes-python]="GET"
|
|
)
|
|
|
|
declare -A CHECK_URL=(
|
|
[hello-node]="https://sless-api.kube5s.ru/fn/default/hello-http"
|
|
[simple-node]="https://sless-api.kube5s.ru/fn/default/simple-node-time-display"
|
|
[simple-python]="https://sless-api.kube5s.ru/fn/default/simple-py-time-display"
|
|
[notes-python]="https://sless-api.kube5s.ru/fn/default/notes-list"
|
|
)
|
|
|
|
declare -A CHECK_DATA=(
|
|
[hello-node]='{"name":"Smoke"}'
|
|
[simple-node]=''
|
|
[simple-python]=''
|
|
[notes-python]=''
|
|
)
|
|
|
|
declare -a PREEXISTING_EXAMPLES=()
|
|
LAST_LOG_FILE=""
|
|
CURRENT_EXAMPLE=""
|
|
CURRENT_STEP=""
|
|
LAST_ERROR_SUMMARY=""
|
|
|
|
fail_run() {
|
|
local example="$1"
|
|
local step="$2"
|
|
local details="$3"
|
|
|
|
echo
|
|
echo "ERROR SUMMARY"
|
|
echo "example: $example"
|
|
echo "step: $step"
|
|
echo "reason: $details"
|
|
|
|
if [ -n "$LAST_LOG_FILE" ] && [ -f "$LAST_LOG_FILE" ]; then
|
|
echo "log: $LAST_LOG_FILE"
|
|
echo "last log lines:"
|
|
tail -n 20 "$LAST_LOG_FILE"
|
|
fi
|
|
|
|
exit 1
|
|
}
|
|
|
|
run_step() {
|
|
local example="$1"
|
|
local step="$2"
|
|
shift 2
|
|
|
|
CURRENT_EXAMPLE="$example"
|
|
CURRENT_STEP="$step"
|
|
LAST_ERROR_SUMMARY=""
|
|
|
|
if ! "$@"; then
|
|
local details="$LAST_ERROR_SUMMARY"
|
|
if [ -z "$details" ]; then
|
|
details="step failed without explicit summary"
|
|
fi
|
|
fail_run "$example" "$step" "$details"
|
|
fi
|
|
}
|
|
|
|
restore_any_backups() {
|
|
local backup
|
|
while IFS= read -r backup; do
|
|
[ -n "$backup" ] || continue
|
|
if [ -f "$backup" ]; then
|
|
mv "$backup" "${backup%.copilot.bak}"
|
|
fi
|
|
done < <(find "$ROOT_DIR" -name '*.copilot.bak' | sort)
|
|
}
|
|
|
|
trap restore_any_backups EXIT
|
|
|
|
clean_local_artifacts() {
|
|
local example="$1"
|
|
rm -rf \
|
|
"$ROOT_DIR/$example/.terraform" \
|
|
"$ROOT_DIR/$example/.terraform.lock.hcl" \
|
|
"$ROOT_DIR/$example/terraform.tfstate" \
|
|
"$ROOT_DIR/$example/terraform.tfstate.backup" \
|
|
"$ROOT_DIR/$example"/terraform.tfstate.*.backup \
|
|
"$ROOT_DIR/$example/dist"
|
|
}
|
|
|
|
retry_tf() {
|
|
local example="$1"
|
|
local label="$2"
|
|
shift 2
|
|
|
|
local attempt=1
|
|
while [ "$attempt" -le 3 ]; do
|
|
LAST_LOG_FILE="$LOG_DIR/${example//\//_}-${label// /_}-${attempt}.log"
|
|
echo "==> [$example] $label (attempt $attempt/3)"
|
|
|
|
(
|
|
cd "$ROOT_DIR/$example"
|
|
"$@"
|
|
) 2>&1 | tee "$LAST_LOG_FILE"
|
|
|
|
local status=${PIPESTATUS[0]}
|
|
if [ "$status" -eq 0 ]; then
|
|
return 0
|
|
fi
|
|
|
|
if grep -Eiq 'Unauthorized|401|403' "$LAST_LOG_FILE"; then
|
|
LAST_ERROR_SUMMARY="authorization error during $label"
|
|
echo "[$example] authorization error during $label"
|
|
return 41
|
|
fi
|
|
|
|
if grep -Eiq 'TLS handshake timeout|tls:.*timeout|i/o timeout|Client\.Timeout exceeded while awaiting headers|context deadline exceeded|unexpected EOF' "$LAST_LOG_FILE" && [ "$attempt" -lt 3 ]; then
|
|
attempt=$((attempt + 1))
|
|
sleep 2
|
|
continue
|
|
fi
|
|
|
|
if grep -Eiq 'TLS handshake timeout|tls:.*timeout|i/o timeout|Client\.Timeout exceeded while awaiting headers|context deadline exceeded|unexpected EOF' "$LAST_LOG_FILE"; then
|
|
LAST_ERROR_SUMMARY="network/provider download failure during $label after retries"
|
|
else
|
|
LAST_ERROR_SUMMARY="terraform command failed during $label with exit code $status"
|
|
fi
|
|
|
|
return "$status"
|
|
done
|
|
|
|
LAST_ERROR_SUMMARY="terraform command failed during $label after exhausting retries"
|
|
return 1
|
|
}
|
|
|
|
record_preexisting_if_needed() {
|
|
local example="$1"
|
|
if grep -Fq 'No changes. Your infrastructure matches the configuration.' "$LAST_LOG_FILE"; then
|
|
PREEXISTING_EXAMPLES+=("$example")
|
|
echo "[$example] detected preexisting remote resources on clean apply"
|
|
fi
|
|
}
|
|
|
|
probe_endpoint() {
|
|
local example="$1"
|
|
local body_file="$LOG_DIR/${example//\//_}-endpoint-body.txt"
|
|
local status_file="$LOG_DIR/${example//\//_}-endpoint-status.txt"
|
|
local method="${CHECK_METHOD[$example]}"
|
|
local url="${CHECK_URL[$example]}"
|
|
local data="${CHECK_DATA[$example]}"
|
|
|
|
if [ "$method" = "POST" ]; then
|
|
curl -sS -X POST -H 'Content-Type: application/json' -d "$data" -o "$body_file" -w '%{http_code}' "$url" > "$status_file"
|
|
else
|
|
curl -sS -o "$body_file" -w '%{http_code}' "$url" > "$status_file"
|
|
fi
|
|
}
|
|
|
|
assert_live_endpoint() {
|
|
local example="$1"
|
|
probe_endpoint "$example"
|
|
|
|
local body_file="$LOG_DIR/${example//\//_}-endpoint-body.txt"
|
|
local status
|
|
status="$(cat "$LOG_DIR/${example//\//_}-endpoint-status.txt")"
|
|
|
|
if [ "$status" != "200" ]; then
|
|
LAST_ERROR_SUMMARY="live endpoint check failed with HTTP $status"
|
|
echo "[$example] live endpoint check failed with HTTP $status"
|
|
cat "$body_file"
|
|
return 1
|
|
fi
|
|
|
|
if grep -Fq 'function unreachable' "$body_file"; then
|
|
LAST_ERROR_SUMMARY="live endpoint returned function unreachable"
|
|
echo "[$example] live endpoint check returned unreachable function"
|
|
cat "$body_file"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
assert_destroyed_endpoint() {
|
|
local example="$1"
|
|
probe_endpoint "$example"
|
|
|
|
local body_file="$LOG_DIR/${example//\//_}-endpoint-body.txt"
|
|
local status
|
|
status="$(cat "$LOG_DIR/${example//\//_}-endpoint-status.txt")"
|
|
|
|
if [ "$status" = "404" ] || [ "$status" = "000" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if grep -Eiq 'not found|404 page not found' "$body_file"; then
|
|
return 0
|
|
fi
|
|
|
|
if [ "$status" = "502" ] && grep -Fq 'function unreachable' "$body_file"; then
|
|
LAST_ERROR_SUMMARY="route cleanup bug: public endpoint still exists but backend is already gone (HTTP 502 function unreachable)"
|
|
echo "[$example] route still exists after destroy, but backend is already gone (HTTP 502 function unreachable)"
|
|
cat "$body_file"
|
|
return 1
|
|
fi
|
|
|
|
LAST_ERROR_SUMMARY="endpoint still responds after destroy with HTTP $status"
|
|
echo "[$example] endpoint still responds after destroy with HTTP $status"
|
|
cat "$body_file"
|
|
return 1
|
|
}
|
|
|
|
wait_for_destroyed_endpoint() {
|
|
local example="$1"
|
|
local attempts=24
|
|
local sleep_sec=5
|
|
local try=1
|
|
|
|
while [ "$try" -le "$attempts" ]; do
|
|
if assert_destroyed_endpoint "$example"; then
|
|
echo "[$example] endpoint disappeared after destroy"
|
|
return 0
|
|
fi
|
|
|
|
echo "[$example] endpoint still present after destroy, waiting (${try}/${attempts})"
|
|
try=$((try + 1))
|
|
sleep "$sleep_sec"
|
|
done
|
|
|
|
echo "[$example] endpoint did not disappear after destroy within $((attempts * sleep_sec))s"
|
|
if [ -z "$LAST_ERROR_SUMMARY" ]; then
|
|
LAST_ERROR_SUMMARY="endpoint remained reachable for more than $((attempts * sleep_sec))s after destroy"
|
|
else
|
|
LAST_ERROR_SUMMARY="$LAST_ERROR_SUMMARY; endpoint was still published after $((attempts * sleep_sec))s"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
backup_and_modify() {
|
|
local example="$1"
|
|
case "$example" in
|
|
hello-node)
|
|
cp "$ROOT_DIR/$example/http.tf" "$ROOT_DIR/$example/http.tf.copilot.bak"
|
|
perl -0pi -e 's/enabled\s+=\s+true/enabled = false/' "$ROOT_DIR/$example/http.tf"
|
|
;;
|
|
simple-node)
|
|
cp "$ROOT_DIR/$example/time-display.tf" "$ROOT_DIR/$example/time-display.tf.copilot.bak"
|
|
perl -0pi -e 's/memory_mb\s+=\s+64/memory_mb = 96/' "$ROOT_DIR/$example/time-display.tf"
|
|
;;
|
|
simple-python)
|
|
cp "$ROOT_DIR/$example/time-display.tf" "$ROOT_DIR/$example/time-display.tf.copilot.bak"
|
|
perl -0pi -e 's/memory_mb\s+=\s+64/memory_mb = 96/' "$ROOT_DIR/$example/time-display.tf"
|
|
;;
|
|
notes-python)
|
|
cp "$ROOT_DIR/$example/notes-list.tf" "$ROOT_DIR/$example/notes-list.tf.copilot.bak"
|
|
perl -0pi -e 's/memory_mb\s+=\s+128/memory_mb = 160/' "$ROOT_DIR/$example/notes-list.tf"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
restore_modified_files() {
|
|
local example="$1"
|
|
case "$example" in
|
|
hello-node)
|
|
mv "$ROOT_DIR/$example/http.tf.copilot.bak" "$ROOT_DIR/$example/http.tf"
|
|
;;
|
|
simple-node)
|
|
mv "$ROOT_DIR/$example/time-display.tf.copilot.bak" "$ROOT_DIR/$example/time-display.tf"
|
|
;;
|
|
simple-python)
|
|
mv "$ROOT_DIR/$example/time-display.tf.copilot.bak" "$ROOT_DIR/$example/time-display.tf"
|
|
;;
|
|
notes-python)
|
|
mv "$ROOT_DIR/$example/notes-list.tf.copilot.bak" "$ROOT_DIR/$example/notes-list.tf"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
run_example() {
|
|
local example="$1"
|
|
|
|
echo
|
|
echo "==== $example ===="
|
|
|
|
clean_local_artifacts "$example"
|
|
run_step "$example" "terraform init" retry_tf "$example" "terraform init" terraform init -input=false -no-color
|
|
run_step "$example" "terraform apply clean" retry_tf "$example" "terraform apply clean" terraform apply -auto-approve -input=false -no-color
|
|
record_preexisting_if_needed "$example"
|
|
run_step "$example" "endpoint check after clean apply" assert_live_endpoint "$example"
|
|
|
|
run_step "$example" "terraform destroy clean" retry_tf "$example" "terraform destroy clean" terraform destroy -auto-approve -input=false -no-color
|
|
run_step "$example" "endpoint cleanup after clean destroy" wait_for_destroyed_endpoint "$example"
|
|
|
|
run_step "$example" "terraform apply second" retry_tf "$example" "terraform apply second" terraform apply -auto-approve -input=false -no-color
|
|
run_step "$example" "endpoint check after second apply" assert_live_endpoint "$example"
|
|
|
|
backup_and_modify "$example"
|
|
run_step "$example" "terraform apply modified" retry_tf "$example" "terraform apply modified" terraform apply -auto-approve -input=false -no-color
|
|
restore_modified_files "$example"
|
|
|
|
run_step "$example" "terraform destroy final" retry_tf "$example" "terraform destroy final" terraform destroy -auto-approve -input=false -no-color
|
|
run_step "$example" "endpoint cleanup after final destroy" wait_for_destroyed_endpoint "$example"
|
|
clean_local_artifacts "$example"
|
|
}
|
|
|
|
main() {
|
|
local example
|
|
for example in "${EXAMPLES[@]}"; do
|
|
run_example "$example"
|
|
done
|
|
|
|
if [ "${#PREEXISTING_EXAMPLES[@]}" -gt 0 ]; then
|
|
echo
|
|
echo "Preexisting remote resources were detected on first apply for: ${PREEXISTING_EXAMPLES[*]}"
|
|
exit 2
|
|
fi
|
|
|
|
echo
|
|
echo "All Terraform example lifecycles completed successfully."
|
|
}
|
|
|
|
main "$@" |