terraform.mk: make tf-init resilient to provider-registry outages

TF_BUILD_DIR ?= .tf_build

TF_ENVVARS_FILE ?= ${TF_BUILD_DIR}/.tf-envvars.sh

# Cache downloaded providers across runs and projects, so a flaky registry only

# has to be reachable once per provider version. On later inits Terraform reuses

# the local copy instead of re-downloading (the ${TF_BUILD_DIR} is wiped every

# render, which would otherwise re-fetch every time). The dir is created in

# tf-init; Terraform itself will not create it.

TF_PLUGIN_CACHE_DIR ?= ${HOME}/.terraform.d/plugin-cache

# `terraform init` reaches out to the provider registry, which periodically

# returns transient 5xx/429/network errors that otherwise abort the whole run.

# Wrap it in retry.sh so those hiccups self-recover; the retry-on pattern keeps

# genuine errors (e.g. backend auth 403) failing fast. Backoff doubles each

# attempt (delay, 2*delay, 4*delay, ...) to ride out longer outages and let

# 429 rate-limits cool down.

TF_INIT_RETRY_ATTEMPTS ?= 4

TF_INIT_RETRY_DELAY ?= 10

TF_INIT_RETRY_ON ?= 5[0-9][0-9] |429|Bad Gateway|Service Unavailable|Gateway Time-?out|Too Many Requests|rate limit|could not query provider registry|failed to install provider|connection reset|connection refused|TLS handshake|i/o timeout|timeout|temporary failure

# Local provider mirror — a filesystem copy of provider plugins that lets

# `terraform init` install entirely from disk, never touching the registry.

# Unlike the plugin cache (which still needs the registry to *select* versions),

# a mirror makes init fully registry-independent. Workflow:

#   make tf-mirror              # populate the mirror while the registry is healthy

#   make plan TF_OFFLINE=1      # init from the mirror during a registry outage

# When TF_OFFLINE is set, tf-init writes a CLI config pointing Terraform at the

# mirror and excluding direct registry installs.

TF_PROVIDER_MIRROR_DIR ?= ${HOME}/.terraform.d/plugin-mirror

TF_CLI_CONFIG_FILE ?= ${CURDIR}/${TF_BUILD_DIR}/.terraformrc

# Platforms to fetch when populating the mirror (CI runners are usually linux).

TF_MIRROR_PLATFORMS ?= darwin_amd64 linux_amd64

ifdef TF_OFFLINE

export TF_CLI_CONFIG_FILE

endif

# set the default tf backend

TF_BACKEND_TYPE ?= gitlab

ifeq ($(strip $(TF_BACKEND_TYPE)),gitlab)

tf-render: tf-check-git-ignore ## render all templates defined in ${TF_DIRS} to build dir

	@if [ "$(MAKELEVEL)" -eq "0" ]; then \

		rm -rf ${TF_BUILD_DIR}/* ; \

		rm -f ${TF_BUILD_DIR}/.terraform.lock.hcl ; \

		if [ -z "$${CI}" ] && [ -z "$${GOOGLE_ENCRYPTION_KEY}" ] && [ -n "${TF_KEY_VAULT_PATH}" ]; then \

			export GOOGLE_ENCRYPTION_KEY="$$(vault-read.sh ${TF_KEY_VAULT_PATH})" ; \

		fi ; \

.PHONY: tf-init

tf-init: tf-env tf-render ## terraform init

	cd ${TF_BUILD_DIR}; ${TF_INIT_CMD}

	@mkdir -p ${TF_PLUGIN_CACHE_DIR}

	@if [ -n "$${TF_OFFLINE}" ]; then \

		printf 'provider_installation {\n  filesystem_mirror {\n    path    = "%s"\n    include = ["registry.terraform.io/*/*"]\n  }\n  direct {\n    exclude = ["registry.terraform.io/*/*"]\n  }\n}\n' "${TF_PROVIDER_MIRROR_DIR}" > ${TF_CLI_CONFIG_FILE}; \

		echo "tf-init: TF_OFFLINE set -> installing providers from mirror ${TF_PROVIDER_MIRROR_DIR}"; \

	fi

	cd ${TF_BUILD_DIR}; ${OTICA_SCRIPTS_DIR}/retry.sh -n ${TF_INIT_RETRY_ATTEMPTS} -d ${TF_INIT_RETRY_DELAY} -m '${TF_INIT_RETRY_ON}' -- ${TF_INIT_CMD}

	@if [ ! -z "${TERRATAG_TAGS}" ]; then terratag ; fi

	@cd ${TF_BUILD_DIR}; ${TF_VALIDATE_CMD}

.PHONY: tf-mirror

tf-mirror: tf-init ## populate the local provider mirror (run while the registry is healthy)

	@mkdir -p ${TF_PROVIDER_MIRROR_DIR}

	cd ${TF_BUILD_DIR}; ${TF_CMD} providers mirror $(addprefix -platform=,${TF_MIRROR_PLATFORMS}) ${TF_PROVIDER_MIRROR_DIR}

	@echo "tf-mirror: mirrored providers to ${TF_PROVIDER_MIRROR_DIR}"

.PHONY: tf-init-get

tf-init-get: TF_INIT_CMD := ${TF_INIT_CMD} -get

tf-init-get: tf-init ## terraform init -get for update the modules

#!/usr/bin/env bash

#

# retry.sh -- run a command, retrying on transient failure with exponential backoff.

#

# Usage: retry.sh [-n attempts] [-d delay] [-m pattern] -- cmd [args...]

#

#   -n attempts   max attempts                 (default ${RETRY_ATTEMPTS:-3})

#   -d delay      base delay seconds, doubles  (default ${RETRY_DELAY:-5})

#                 after each failed attempt

#   -m pattern    retry-on egrep pattern       (default ${RETRY_ON:-})

#                 When set, only retry if the command's output matches the

#                 pattern (case-insensitive); any other failure fails fast.

#                 When unset, retry on any non-zero exit.

#

# Exits with the last attempt's exit code. Combined stdout+stderr of the command

# is streamed live and also scanned against the retry-on pattern, so this works

# both interactively and in CI (no TTY required).

#

# Example -- ride out registry.terraform.io 5xx hiccups during init:

#   retry.sh -m '502|503|Bad Gateway|could not query provider registry' -- terraform init

set -uo pipefail

attempts=${RETRY_ATTEMPTS:-3}

delay=${RETRY_DELAY:-5}

pattern=${RETRY_ON:-}

while getopts ":n:d:m:" opt; do

    case "$opt" in

        n) attempts=$OPTARG ;;

        d) delay=$OPTARG ;;

        m) pattern=$OPTARG ;;

        :) echo "retry.sh: option -$OPTARG requires an argument" >&2; exit 2 ;;

        \?) echo "retry.sh: invalid option -$OPTARG" >&2; exit 2 ;;

    esac

done

shift $((OPTIND - 1))

[ "${1:-}" = "--" ] && shift

[ "$#" -gt 0 ] || { echo "retry.sh: no command given" >&2; exit 2; }

log=$(mktemp "${TMPDIR:-/tmp}/retry.XXXXXX")

trap 'rm -f "$log"' EXIT

attempt=1

while :; do

    "$@" 2>&1 | tee "$log"

    rc=${PIPESTATUS[0]}

    [ "$rc" -eq 0 ] && exit 0

    if [ "$attempt" -ge "$attempts" ]; then

        echo "retry.sh: command failed after ${attempt} attempt(s) (exit ${rc})" >&2

        exit "$rc"

    fi

    if [ -n "$pattern" ] && ! grep -Eqi -- "$pattern" "$log"; then

        echo "retry.sh: failure does not match retry pattern; not retrying" >&2

        exit "$rc"

    fi

    echo "retry.sh: attempt ${attempt}/${attempts} failed (exit ${rc}); retrying in ${delay}s..." >&2

    sleep "$delay"

    attempt=$((attempt + 1))

    delay=$((delay * 2))

done