From 22ac3af36504378cc489cc798758bf8c9e7e648c Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:21:45 +1100 Subject: [PATCH 01/40] [#154] Added Google Drive database fetch and push scripts and the 'vortex-push-db' router. --- .vortex/tooling/composer.json | 1 + .vortex/tooling/src/vortex-fetch-db | 3 +- .vortex/tooling/src/vortex-fetch-db-gdrive | 135 ++++++++ .vortex/tooling/src/vortex-push-db | 54 ++++ .vortex/tooling/src/vortex-push-db-gdrive | 160 ++++++++++ .vortex/tooling/src/vortex-push-db-s3 | 18 +- .vortex/tooling/src/vortex-refresh-db | 49 ++- .vortex/tooling/tests/_helper.bash | 1 + .../tooling/tests/unit/fetch-db-gdrive.bats | 293 +++++++++++++++++ .vortex/tooling/tests/unit/fetch-db.bats | 9 +- .../tooling/tests/unit/push-db-gdrive.bats | 297 ++++++++++++++++++ .vortex/tooling/tests/unit/push-db-s3.bats | 60 ++++ .vortex/tooling/tests/unit/push-db.bats | 125 ++++++++ .vortex/tooling/tests/unit/refresh-db.bats | 180 ++++++++++- 14 files changed, 1369 insertions(+), 16 deletions(-) create mode 100755 .vortex/tooling/src/vortex-fetch-db-gdrive create mode 100755 .vortex/tooling/src/vortex-push-db create mode 100755 .vortex/tooling/src/vortex-push-db-gdrive create mode 100644 .vortex/tooling/tests/unit/fetch-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db.bats diff --git a/.vortex/tooling/composer.json b/.vortex/tooling/composer.json index 492b6c507a..662ed2da14 100644 --- a/.vortex/tooling/composer.json +++ b/.vortex/tooling/composer.json @@ -35,6 +35,7 @@ "src/vortex-logout", "src/vortex-notify", "src/vortex-provision", + "src/vortex-push-db", "src/vortex-push-db-image", "src/vortex-refresh-db", "src/vortex-reset", diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index 1cc7d7acf7..faf32b4ac9 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -64,8 +64,9 @@ case "${VORTEX_FETCH_DB_SOURCE}" in lagoon) fetch_script="vortex-fetch-db-lagoon" ;; container_registry) fetch_script="vortex-fetch-db-container-registry" ;; s3) fetch_script="vortex-fetch-db-s3" ;; + gdrive) fetch_script="vortex-fetch-db-gdrive" ;; none) fetch_script="" ;; - *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none." ;; + *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none." ;; esac # Skip file existence check for container_registry source as the database is diff --git a/.vortex/tooling/src/vortex-fetch-db-gdrive b/.vortex/tooling/src/vortex-fetch-db-gdrive new file mode 100755 index 0000000000..546b40dc53 --- /dev/null +++ b/.vortex/tooling/src/vortex-fetch-db-gdrive @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +## +# Fetch DB dump from Google Drive. +# +# Authenticates with a Google Cloud service account key and downloads the file +# with the database dump file name from a Google Drive folder. The folder may +# be in a shared drive. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant. +_db_index="${VORTEX_DB_INDEX:-}" + +# Google Cloud service account key: the JSON content or a path to the JSON +# key file. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY" +VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${!_v:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" + +# Google Drive folder ID with the database dump file. +# +# The ID is the last segment of the folder URL. The folder must be shared with +# the service account. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID" +VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${!_v:-}" + +# Directory with database dump file. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_FETCH_DB_GDRIVE_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" + +# Database dump file name. +# +# The same name is used for the file in the Google Drive folder. +_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_FETCH_DB_GDRIVE_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +extract_json_last_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +extract_json_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +# Print a Google OAuth access token for a service account key and a scope. +# +# The key is passed to PHP in the environment, so it never appears in the +# process list. +gdrive_access_token() { + local key="${1}" + local scope="${2}" + + [ -f "${key}" ] && key="$(cat "${key}")" + + local assertion + # shellcheck disable=SC2016 + assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 + + local token_json + token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 + + echo "${token_json}" | extract_json_value "access_token" +} + +# Print the ID of the most recently modified file with a name in a folder. +# +# Prints nothing when the folder has no such file, and fails when the folder +# cannot be listed. +gdrive_find_file() { + local token="${1}" + local folder_id="${2}" + local name="${3}" + + local files_json + files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 + + local files + files=$(echo "${files_json}" | extract_json_value "files") || return 1 + + echo "${files}" | extract_json_last_value "id" || true +} + +for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." +[ -z "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID." + +info "Started database dump fetch from Google Drive." + +note "Remote file: ${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" +note "Local path: ${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" +note "Folder ID: ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" + +task "Authenticating with the service account." +token=$(gdrive_access_token "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive.readonly") || fail "Unable to authenticate with the service account. Check that VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." +pass "Authenticated with the service account." + +task "Finding the database dump file in the folder." +file_id=$(gdrive_find_file "${token}" "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_FETCH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." +[ -z "${file_id}" ] && fail "Unable to find the ${VORTEX_FETCH_DB_GDRIVE_DB_FILE} file in the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder." +pass "Found the database dump file with ID ${file_id}." + +mkdir -p "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}" + +task "Downloading the database dump file." +curl -sS -f -L -H "Authorization: Bearer ${token}" -o "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" "https://www.googleapis.com/drive/v3/files/${file_id}?alt=media&supportsAllDrives=true" || fail "Unable to download the database dump file." +pass "Downloaded the database dump file." + +# Enable xtrace only after the authenticated requests so the access token is +# never written to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +pass "Finished database dump fetch from Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db new file mode 100755 index 0000000000..4ba20e8c4d --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +## +# Push database dump. +# +# This is a router script to call relevant scripts based on type. +# +# The database dump file is pushed to the storage the database is fetched +# from, set in VORTEX_FETCH_DB_SOURCE. Only the 'gdrive' and 's3' sources +# accept a push; the push is skipped for other sources. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Proceed with push. +# +# Set to 1 in the nightly CI job only, so the storage is refreshed once a day +# and never from a feature branch build. +VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" + +# Database fetch source, which is also the push destination. +VORTEX_FETCH_DB_SOURCE="${VORTEX_FETCH_DB_SOURCE:-url}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +info "Started database push." + +[ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 + +case "${VORTEX_FETCH_DB_SOURCE}" in + gdrive) push_script="vortex-push-db-gdrive" ;; + s3) push_script="vortex-push-db-s3" ;; + *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." && exit 0 ;; +esac + +# The pushed dump leaves the hosting and CI, so it must be sanitized. +[ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + +"$(dirname "${BASH_SOURCE[0]}")/${push_script}" + +pass "Finished database push." diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive new file mode 100755 index 0000000000..0f153bc016 --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db-gdrive @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +## +# Push DB dump to Google Drive. +# +# Authenticates with a Google Cloud service account key and uploads the +# database dump file to a Google Drive folder. A file with the same name in +# the folder is replaced in place, so its ID and sharing settings are kept. +# +# Service accounts have no storage quota of their own, so the folder must be +# in a shared drive. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu +set -o pipefail + +# Google Cloud service account key: the JSON content or a path to the JSON +# key file. +VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" + +# Google Drive folder ID to push the database dump file to. +# +# Defaults to the folder the database is fetched from. +VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID:-${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID:-}}" + +# Directory with database dump file. +VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" + +# Database dump file name. +# +# The same name is used for the file in the Google Drive folder. +VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +extract_json_last_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +extract_json_value() { + local key="${1}" + php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" +} + +# Print a Google OAuth access token for a service account key and a scope. +# +# The key is passed to PHP in the environment, so it never appears in the +# process list. +gdrive_access_token() { + local key="${1}" + local scope="${2}" + + [ -f "${key}" ] && key="$(cat "${key}")" + + local assertion + # shellcheck disable=SC2016 + assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 + + local token_json + token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 + + echo "${token_json}" | extract_json_value "access_token" +} + +# Print the ID of the most recently modified file with a name in a folder. +# +# Prints nothing when the folder has no such file, and fails when the folder +# cannot be listed. +gdrive_find_file() { + local token="${1}" + local folder_id="${2}" + local name="${3}" + + local files_json + files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 + + local files + files=$(echo "${files_json}" | extract_json_value "files") || return 1 + + echo "${files}" | extract_json_last_value "id" || true +} + +for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." +[ -z "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + +local_file="${VORTEX_PUSH_DB_GDRIVE_DB_DIR}/${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" +[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." + +info "Started database dump push to Google Drive." + +note "Local file: ${local_file}" +note "Remote file: ${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" +note "Folder ID: ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" + +task "Authenticating with the service account." +token=$(gdrive_access_token "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive") || fail "Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." +pass "Authenticated with the service account." + +file_id=$(gdrive_find_file "${token}" "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_PUSH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." + +upload_url="https://www.googleapis.com/upload/drive/v3/files" +if [ -n "${file_id}" ]; then + note "Replacing the existing file with ID ${file_id}." + request_type="PATCH" + upload_url="${upload_url}/${file_id}" + metadata="{}" +else + note "Creating a new file in the folder." + request_type="POST" + metadata=$(GDRIVE_NAME="${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" php -r 'print json_encode(["name" => getenv("GDRIVE_NAME"), "parents" => [getenv("GDRIVE_FOLDER_ID")]]);') +fi + +# A resumable upload session accepts the whole file in a single request, so +# the dump is not limited by the 5 MB multipart upload size. +task "Starting the upload session." +session_headers=$(curl -sS -X "${request_type}" -D - -o /dev/null -H "Authorization: Bearer ${token}" -H "Content-Type: application/json; charset=UTF-8" -H "X-Upload-Content-Type: application/octet-stream" --data "${metadata}" "${upload_url}?uploadType=resumable&supportsAllDrives=true") || fail "Unable to start the upload session." +session_url=$(echo "${session_headers}" | sed -n 's/^[Ll]ocation: *//p' | tr -d '\r') +[ -z "${session_url}" ] && fail "Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." +pass "Started the upload session." + +task "Uploading the database dump file." +set +e +response=$(curl -sS -X PUT --upload-file "${local_file}" -H "Content-Type: application/octet-stream" --write-out "\n%{http_code}" "${session_url}") +curl_status=$? +set -e + +# Enable xtrace only after the authenticated requests so the access token and +# the upload session URL are never written to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +[ "${curl_status}" -ne 0 ] && fail "Google Drive upload request failed with curl exit ${curl_status}." + +http_code=$(echo "${response}" | tail -1) +response_body=$(echo "${response}" | sed '$d') + +case "${http_code}" in + 2*) ;; + *) + [ -n "${response_body}" ] && echo "Response: ${response_body}" + fail "Unable to push to Google Drive: HTTP status ${http_code}." + ;; +esac +pass "Uploaded the database dump file." + +pass "Finished database dump push to Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 93470ad6ed..569555f344 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -13,20 +13,23 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu set -o pipefail +# The S3 values fall back to the values the database is fetched with, so a +# bucket used as the fetch source is configured once. + # AWS access key. -VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${VORTEX_FETCH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}}" # AWS secret key. -VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}" +VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${VORTEX_FETCH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}}" # S3 bucket name. -VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${S3_BUCKET:-}}" +VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${VORTEX_FETCH_DB_S3_BUCKET:-${S3_BUCKET:-}}}" # S3 region. -VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${S3_REGION:-}}" +VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${VORTEX_FETCH_DB_S3_REGION:-${S3_REGION:-}}}" # S3 prefix (path within the bucket). -VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${S3_PREFIX:-}}" +VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFIX:-${S3_PREFIX:-}}}" # Directory with database dump file. VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" @@ -35,7 +38,10 @@ VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}} VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" # Remote database dump file name. -VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-db.sql}" +# +# Defaults to the database dump file name, which is the name the database is +# fetched with. +VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-${VORTEX_PUSH_DB_S3_DB_FILE}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index bc8ee64a8c..9b2f01e2e5 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,6 +15,11 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # +# With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED +# set to 1, the dump file is fetched from VORTEX_REFRESH_DB_SOURCE before +# provisioning and the database is sanitized after it, so the exported dump +# can be pushed to the store with 'vortex-push-db'. +# # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. # @@ -39,12 +44,16 @@ _db_index="${VORTEX_DB_INDEX:-}" # VORTEX_DB2_IMAGE_BASE and falls back to this value. VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" -# Fetch source of the database dump file to import into the base image. +# Upstream fetch source of the database dump file. # # Required when the fetch source is 'container_registry' and a base image is # set: the database image is built from a dump file, so it needs a dump file -# source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed -# database uses VORTEX_REFRESH_DB2_SOURCE. +# source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An +# indexed database uses VORTEX_REFRESH_DB2_SOURCE. +# +# Also required when the fetch source is a dump store ('gdrive' or 's3') and +# VORTEX_PUSH_DB_PROCEED is set to 1: the dump pushed to the store comes from +# an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -98,14 +107,40 @@ fi rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 +# A dump store ('gdrive' or 's3') receives the dump this refresh exports, so +# with a push requested the dump is fetched again from its upstream source. +refresh_store=0 +if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + case "${fetch_source}" in + gdrive | s3) refresh_store=1 ;; + esac +fi + +if [ "${refresh_store}" -eq 1 ]; then + case "${VORTEX_REFRESH_DB_SOURCE}" in + "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${fetch_source} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + esac + [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." +fi + if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." ;; + "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." ;; esac fi "$(dirname "${BASH_SOURCE[0]}")/vortex-login-container-registry" +if [ "${refresh_store}" -eq 1 ]; then + # The dump fetched from the store is removed, so the check below proves the + # upstream source wrote a new one. + rm -f "${db_file}" + # CircleCI re-sources the job variables from BASH_ENV in every bash + # script, which would replace the prefixed values. + env BASH_ENV='' "VORTEX_FETCH_DB_SOURCE=${VORTEX_REFRESH_DB_SOURCE}" "VORTEX_FETCH_DB_FORCE=1" "$(dirname "${BASH_SOURCE[0]}")/vortex-fetch-db" + [ -f "${db_file}" ] || fail "The database dump file was not fetched to ${db_file}. Remove the output directory and file name overrides of the database fetch source." +fi + # The value of VORTEX_DB_IMAGE passed to the provisioning. An empty value # makes the provisioning import the dump file. provision_db_image="${db_image}" @@ -160,6 +195,12 @@ else docker compose exec "${env_args[@]}" -e "VORTEX_DB_IMAGE=${provision_db_image}" -e VORTEX_PROVISION_POST_OPERATIONS_SKIP=1 -T cli ./vendor/bin/vortex-provision + # The provisioning skips the post-provision operations, which include the + # sanitization, so the dump leaves the hosting and CI only after this step. + if [ "${refresh_store}" -eq 1 ]; then + docker compose exec "${env_args[@]}" -T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db || fail "Unable to sanitize the database." + fi + # Database-in-image storage exports the image archive instead of the dump. if [ -n "${db_image}" ]; then rm -f "${db_file}" diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 4cbded3356..61bb8a2d46 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -121,6 +121,7 @@ setup() { unset VORTEX_DB_IMAGE unset VORTEX_FETCH_DB_FORCE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED + unset VORTEX_PUSH_DB_PROCEED # Isolate ambient credentials from the scripts under test. Scripts fall back # to these unprefixed values whenever their own variable is unset or empty, diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats new file mode 100644 index 0000000000..863221bf25 --- /dev/null +++ b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats @@ -0,0 +1,293 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-fetch-db-gdrive' script. +# +# shellcheck disable=SC2016,SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Writes a service account key file with a newly generated private key. +fixture_gdrive_key() { + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null + php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" +} + +@test "fetch-db-gdrive: Fetch database file successfully with a key file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token","expires_in":3599}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_side_effect "${mock_curl}" "echo 'CREATE TABLE test (id INT);' > .data/db.sql" 3 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + assert_output_contains "[INFO] Started database dump fetch from Google Drive." + assert_output_contains "Remote file: db.sql" + assert_output_contains "Local path: .data/db.sql" + assert_output_contains "Folder ID: folder-123" + assert_output_contains "[ OK ] Authenticated with the service account." + assert_output_contains "[ OK ] Found the database dump file with ID file-456." + assert_output_contains "[ OK ] Downloaded the database dump file." + assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." + assert_output_not_contains "test-token" + + assert_equal "3" "$(mock_get_call_num "${mock_curl}")" + + # The token request carries a signed JWT assertion for the read-only scope. + token_args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${token_args}" "https://oauth2.googleapis.com/token" + assert_string_contains "${token_args}" "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" + assertion="${token_args##*assertion=}" + assertion="${assertion%% *}" + IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" + assert_not_empty "${jwt_signature}" + claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" + assert_string_contains "${claims}" '"iss":"vortex@example.iam.gserviceaccount.com"' + assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive.readonly"' + + list_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${list_args}" "https://www.googleapis.com/drive/v3/files" + assert_string_contains "${list_args}" "Authorization: Bearer test-token" + assert_string_contains "${list_args}" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" + assert_string_contains "${list_args}" "supportsAllDrives=true" + + download_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${download_args}" "Authorization: Bearer test-token" + assert_string_contains "${download_args}" "-o .data/db.sql" + assert_string_contains "${download_args}" "https://www.googleapis.com/drive/v3/files/file-456?alt=media&supportsAllDrives=true" + + assert_file_contains ".data/db.sql" "CREATE TABLE test" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fetch database file with the key content in the shared variable" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + unset VORTEX_FETCH_DB_GDRIVE_DB_DIR VORTEX_FETCH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + # The default directory and file name come from .env. + assert_output_contains "Local path: ./.data/db.sql" + assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 3)" "-o ./.data/db.sql" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Resolve indexed variables with VORTEX_DB_INDEX" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB3_GDRIVE_FOLDER_ID="folder-789" + export VORTEX_FETCH_DB3_FILE="db3.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + assert_output_contains "Remote file: db3.sql" + assert_output_contains "Folder ID: folder-789" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db3.sql' and 'folder-789' in parents and trashed = false" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Escape a quote in the file name of the folder query" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="site's.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_success + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'site\\'s.sql' and" + + rm -rf .data + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the service account key is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder ID is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the service account key is invalid" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + for key in "not json" '{"client_email":"vortex@example.iam.gserviceaccount.com"}' '{"client_email":"vortex@example.iam.gserviceaccount.com","private_key":"not a key"}'; do + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${key}" + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." + done + + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the token request is rejected" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"error":"invalid_grant","error_description":"Invalid JWT Signature."}' 1 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account." + assert_output_not_contains "Found the database dump file" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder cannot be listed" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"error":{"code":404,"message":"File not found: folder-123."}}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the folder has no database dump file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to find the db.sql file in the folder-123 folder." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "fetch-db-gdrive: Fail when the download fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + fixture_gdrive_key + + export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 22 3 + + run .vortex/tooling/src/vortex-fetch-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to download the database dump file." + assert_output_not_contains "Finished database dump fetch from Google Drive." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index fec06732f3..c274716c30 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -14,6 +14,7 @@ stub_fetch_scripts() { stub_sibling "vortex-fetch-db-lagoon" "Fetched from lagoon." stub_sibling "vortex-fetch-db-container-registry" "Fetched from container_registry." stub_sibling "vortex-fetch-db-s3" "Fetched from s3." + stub_sibling "vortex-fetch-db-gdrive" "Fetched from gdrive." } @test "fetch-db: Fetch with URL source" { @@ -307,7 +308,7 @@ EOF export VORTEX_FETCH_DB_PROCEED="1" - for db_source in url ftp acquia lagoon container_registry s3; do + for db_source in url ftp acquia lagoon container_registry s3 gdrive; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -315,7 +316,7 @@ EOF assert_output_contains "Fetched from ${db_source}." assert_output_contains "Finished database fetch." - for other_source in url ftp acquia lagoon container_registry s3; do + for other_source in url ftp acquia lagoon container_registry s3 gdrive; do if [ "${other_source}" != "${db_source}" ]; then assert_output_not_contains "Fetched from ${other_source}." fi @@ -357,7 +358,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database fetch." - assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' + assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' assert_output_not_contains "Fetched from" assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" @@ -397,7 +398,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database 2 fetch." - assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' + assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' assert_output_not_contains "Finished database 2 fetch." popd >/dev/null diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats new file mode 100644 index 0000000000..0d7e6b4cc6 --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db-gdrive.bats @@ -0,0 +1,297 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db-gdrive' script. +# +# shellcheck disable=SC2016,SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Writes a service account key file with a newly generated private key. +fixture_gdrive_key() { + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null + php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" +} + +@test "push-db-gdrive: Create the database dump file in the folder" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "CREATE TABLE test (id INT);" >.data/db.sql + fixture_gdrive_key + + unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1\r\ncontent-length: 0\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "[INFO] Started database dump push to Google Drive." + assert_output_contains "Local file: .data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "Folder ID: folder-123" + assert_output_contains "[ OK ] Authenticated with the service account." + assert_output_contains "Creating a new file in the folder." + assert_output_contains "[ OK ] Started the upload session." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Finished database dump push to Google Drive." + assert_output_not_contains "test-token" + assert_output_not_contains "session-1" + + assert_equal "4" "$(mock_get_call_num "${mock_curl}")" + + # The token request carries a signed JWT assertion for the full Drive scope. + token_args="$(mock_get_call_args "${mock_curl}" 1)" + assertion="${token_args##*assertion=}" + assertion="${assertion%% *}" + IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" + assert_not_empty "${jwt_signature}" + claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" + assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive"' + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" + + session_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${session_args}" "-X POST" + assert_string_contains "${session_args}" "Authorization: Bearer test-token" + assert_string_contains "${session_args}" '--data {"name":"db.sql","parents":["folder-123"]}' + assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" + + upload_args="$(mock_get_call_args "${mock_curl}" 4)" + assert_string_contains "${upload_args}" "-X PUT --upload-file .data/db.sql" + assert_string_contains "${upload_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Replace the existing database dump file in place" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY + # The folder falls back to the folder the database is fetched from. + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="fetch-folder" + unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nLocation: https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&upload_id=session-2\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-456"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Folder ID: fetch-folder" + assert_output_contains "Replacing the existing file with ID file-456." + assert_output_contains "[ OK ] Finished database dump push to Google Drive." + + session_args="$(mock_get_call_args "${mock_curl}" 3)" + assert_string_contains "${session_args}" "-X PATCH" + assert_string_contains "${session_args}" "--data {}" + assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&supportsAllDrives=true" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "upload_id=session-2" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the service account key is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the folder ID is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" + export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the database dump file does not exist" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Database dump file .data/db.sql does not exist." + assert_output_not_contains "Started database dump push to Google Drive." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the service account key is invalid" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="not json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail without creating a file when the folder cannot be listed" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"error":{"code":403,"message":"Insufficient permissions."}}' 2 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." + assert_output_not_contains "Creating a new file in the folder." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload session has no location" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 403\r\ncontent-type: application/json\r\n' 3 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." + assert_equal "3" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload returns a non-2xx status" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-3\r\n' 3 + mock_set_output "${mock_curl}" $'{"error":{"code":403,"message":"The user does not have sufficient permissions."}}\n403' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "The user does not have sufficient permissions." + assert_output_contains "[FAIL] Unable to push to Google Drive: HTTP status 403." + assert_output_not_contains "Finished database dump push to Google Drive." + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-gdrive: Fail when the upload request fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-4\r\n' 3 + mock_set_status "${mock_curl}" 28 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_failure + assert_output_contains "[FAIL] Google Drive upload request failed with curl exit 28." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 7f8f96d9c5..6c7ccbe3d3 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -28,6 +28,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -67,6 +68,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -115,6 +117,7 @@ load ../_helper.bash export S3_REGION="us-east-1" # Clear prefix shortcut to prevent environment leakage. unset S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_DB_DIR=".data" export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" @@ -147,6 +150,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -186,6 +190,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -225,6 +230,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -263,6 +269,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -292,6 +299,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -317,6 +325,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="" @@ -342,6 +351,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -367,6 +377,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -395,6 +406,7 @@ load ../_helper.bash # Clear shortcut variables to prevent environment leakage. unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" @@ -411,3 +423,51 @@ load ../_helper.bash popd >/dev/null } + +@test "push-db-s3: Use the database fetch values" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db2.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: .data/db2.sql" + "Remote file: store/db2.sql" + "S3 bucket: fetch-bucket" + "S3 region: eu-west-1" + "S3 prefix: store/" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="" + export VORTEX_PUSH_DB_S3_SECRET_KEY="" + export VORTEX_PUSH_DB_S3_BUCKET="" + export VORTEX_PUSH_DB_S3_REGION="" + export VORTEX_PUSH_DB_S3_PREFIX="" + export VORTEX_FETCH_DB_S3_ACCESS_KEY="fetch-access-key" + export VORTEX_FETCH_DB_S3_SECRET_KEY="fetch-secret-key" + export VORTEX_FETCH_DB_S3_BUCKET="fetch-bucket" + export VORTEX_FETCH_DB_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB_S3_PREFIX="store" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + # The remote file name follows the dump file name. + export VORTEX_PUSH_DB_S3_DB_FILE="db2.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats new file mode 100644 index 0000000000..d2dee534f0 --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -0,0 +1,125 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db' script. +# +# shellcheck disable=SC2030,SC2031 + +load ../_helper.bash + +@test "push-db: Skip when the push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + export VORTEX_FETCH_DB_SOURCE="gdrive" + + for proceed in "" 0; do + export VORTEX_PUSH_DB_PROCEED="${proceed}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database push." + assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." + assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Route each dump store source to its push script" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-s3" "Pushed to s3." + + export VORTEX_PUSH_DB_PROCEED=1 + + for db_source in gdrive s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushed to ${db_source}." + assert_output_contains "[ OK ] Finished database push." + + for other_source in gdrive s3; do + [ "${other_source}" = "${db_source}" ] && continue + assert_output_not_contains "Pushed to ${other_source}." + done + done + + popd >/dev/null +} + +@test "push-db: Skip for a source that does not accept a push" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-s3" "Pushed to s3." + + export VORTEX_PUSH_DB_PROCEED=1 + + for db_source in url ftp acquia lagoon container_registry none; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." + assert_output_not_contains "Pushed to" + assert_output_not_contains "Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Use the source from .env by default" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + unset VORTEX_FETCH_DB_SOURCE + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3.' + assert_output_not_contains "Pushed to gdrive." + + popd >/dev/null +} + +@test "push-db: Refuse to push a database that is not sanitized" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." + assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} + +@test "push-db: Fail when the push script fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + printf '#!/usr/bin/env bash\necho "Failed to push to s3."\nexit 1\n' >.vortex/tooling/src/vortex-push-db-s3 + chmod +x .vortex/tooling/src/vortex-push-db-s3 + + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_PUSH_DB_PROCEED=1 + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains "Failed to push to s3." + assert_output_not_contains "Finished database push." + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 091c08d083..f81a93af38 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -236,7 +236,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." assert_output_not_contains "Logged in to the registry." assert_output_not_contains "Fetched the dump file." done @@ -246,6 +246,184 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Fetches the dump file from the refresh source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + touch "${BUILD_DIR}/bash_env" + export BASH_ENV="${BUILD_DIR}/bash_env" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\necho "Fetched from ${VORTEX_FETCH_DB_SOURCE} with force ${VORTEX_FETCH_DB_FORCE} and BASH_ENV [${BASH_ENV}]."\n' >.vortex/tooling/src/vortex-fetch-db + chmod +x .vortex/tooling/src/vortex-fetch-db + + export VORTEX_FETCH_DB_SOURCE="gdrive" + mkdir -p .data + echo "store" >.data/db.sql + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 + mock_set_output "${mock_docker}" "Sanitized the database." 5 + mock_set_output "${mock_docker}" "Copied the dump file back." 6 + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "Fetched from lagoon with force 1 and BASH_ENV []." + assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." + assert_output_contains "Sanitized the database." + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + # The store holds no database image, so the provisioning imports the dump. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 4)" "-e VORTEX_DB_IMAGE= -e VORTEX_PROVISION_POST_OPERATIONS_SKIP=1 -T cli ./vendor/bin/vortex-provision" + # The provisioning skips the sanitization, so it runs before the export. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "-T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db" + assert_string_contains "$(mock_get_call_args "${mock_docker}" 6)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" + assert_equal "6" "$(mock_get_call_num "${mock_docker}")" + + assert_file_contains .data/db.sql "upstream" + + popd >/dev/null +} + +@test "refresh-db: Provisions from the dump store without the refresh source when a push is not requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "store" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="gdrive" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_not_contains "Fetched the dump file." + assert_output_contains "Exported the database to db.sql." + assert_output_contains "[ OK ] Finished database refresh." + + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + assert_file_contains .data/db.sql "store" + + popd >/dev/null +} + +@test "refresh-db: Fails without an upstream source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + + mock_docker=$(mock_command "docker") + + for refresh_source in "" gdrive s3 container_registry none; do + export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." + assert_output_not_contains "Logged in to the registry." + assert_output_not_contains "Fetched the dump file." + done + + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." + + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_not_contains "Fetched the dump file." + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + +@test "refresh-db: Fails when the sanitization fails for a dump store source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="url" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\n' >.vortex/tooling/src/vortex-fetch-db + chmod +x .vortex/tooling/src/vortex-fetch-db + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + mock_set_status "${mock_docker}" 1 5 + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "[FAIL] Unable to sanitize the database." + assert_output_not_contains "Exported the database to db.sql." + + popd >/dev/null +} + +@test "refresh-db: Fails when the refresh source does not fetch the dump file for a dump store source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "store" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE + export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="url" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file elsewhere." + + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_failure + assert_output_contains "Fetched the dump file elsewhere." + assert_output_contains "[FAIL] The database dump file was not fetched to ./.data/db.sql. Remove the output directory and file name overrides of the database fetch source." + + # The stale dump from the store is never provisioned or pushed. + assert_file_not_exists .data/db.sql + assert_equal "0" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + @test "refresh-db: Fails when the exported dump file cannot be copied from the cli container" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 From 50a1c5683a1463e4055f6a40a6996e7f1b6f1d2e Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:25:21 +1100 Subject: [PATCH 02/40] [#154] Fixed mock expectations in the Google Drive and refresh tests. --- .circleci/config.yml | 16 ++++++++ .env | 37 ++++++++++++++++--- .env.local.example | 8 ++++ .github/workflows/build-test-deploy.yml | 22 +++++++++++ .../Prompts/Handlers/DatabaseFetchSource.php | 10 +++++ .../Prompts/Handlers/MigrationFetchSource.php | 4 ++ .../DatabaseFetchSourceHandlerProcessTest.php | 3 ++ ...MigrationFetchSourceHandlerProcessTest.php | 14 +++++++ .../tooling/tests/unit/fetch-db-gdrive.bats | 3 ++ .vortex/tooling/tests/unit/refresh-db.bats | 4 ++ 10 files changed, 115 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b43026f17..4647de6b11 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -363,6 +363,18 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -387,6 +399,10 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + VORTEX_PUSH_DB_PROCEED: 1 + #;> DB_PUSH # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 #;> !PROVISION_TYPE_PROFILE diff --git a/.env b/.env index 94f358f542..eaa76a2e98 100644 --- a/.env +++ b/.env @@ -222,9 +222,9 @@ VORTEX_FETCH_DB_SOURCE=url # VORTEX_DB_IMAGE=your_org/your_site:latest # Source of the database dump file the nightly CI job imports into a clean -# base image to build the database container image: url, ftp, acquia, lagoon -# or s3. The fetch script of that source reads its own variables, such as -# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# base image to build the database container image: url, ftp, acquia, lagoon, +# s3 or gdrive. The fetch script of that source reads its own variables, such +# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -258,6 +258,24 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 +#;< DB_FETCH_SOURCE_GDRIVE +# Database dump file sourced from Google Drive. + +# Google Drive folder ID with the database dump file. +# +# The ID is the last segment of the folder URL. The folder must be in a shared +# drive and shared with the service account. +VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= +#;> DB_FETCH_SOURCE_GDRIVE + +#;< DB_PUSH +# Source of the database dump file the nightly CI job sanitizes and pushes to +# the database fetch source: url, ftp, acquia or lagoon. The fetch script of +# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. +# Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url +#;> DB_PUSH + # Environment to fetch the database from. # # Applies to hosting environments. @@ -297,9 +315,9 @@ VORTEX_FETCH_DB2_SOURCE=url # Source of the second database dump file the nightly CI job imports into a # clean base image to rebuild the migration database container image: url, -# ftp, acquia, lagoon or s3. The fetch script of that source reads its own -# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets -# in CI. +# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its +# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep +# secrets in CI. # VORTEX_REFRESH_DB2_SOURCE=url #;> MIGRATION_DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -333,6 +351,13 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 +#;< MIGRATION_DB_FETCH_SOURCE_GDRIVE +# Second database dump file sourced from Google Drive. + +# Google Drive folder ID with the second database dump file. +VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= +#;> MIGRATION_DB_FETCH_SOURCE_GDRIVE + # Environment to fetch the second database from. # # Applies to hosting environments. diff --git a/.env.local.example b/.env.local.example index b7a2eb4d0c..e876e08dd8 100644 --- a/.env.local.example +++ b/.env.local.example @@ -69,6 +69,14 @@ VORTEX_FETCH_DB_S3_ACCESS_KEY= VORTEX_FETCH_DB_S3_SECRET_KEY= #;> DB_FETCH_SOURCE_S3 +#;< DB_FETCH_SOURCE_GDRIVE +# Database dump file sourced from Google Drive. +# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key + +# Google Cloud service account key: the JSON content or a path to the JSON key file. +VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= +#;> DB_FETCH_SOURCE_GDRIVE + #;< DB_FETCH_SOURCE_CONTAINER_REGISTRY # Database image sourced from a container registry. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index c770b62aa9..a7eda4a045 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -225,6 +225,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. @@ -281,6 +288,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -343,6 +354,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index 868e643278..6fd4967b04 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -21,6 +21,8 @@ class DatabaseFetchSource extends AbstractHandler { const S3 = 's3'; + const GDRIVE = 'gdrive'; + const NONE = 'none'; /** @@ -55,6 +57,7 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', + self::GDRIVE => 'Google Drive', self::NONE => 'None', ]; @@ -132,6 +135,7 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, + self::GDRIVE, ]; foreach ($types as $type) { @@ -150,6 +154,12 @@ public function process(): void { if ($v !== self::CONTAINER_REGISTRY && !$is_migration_in_image) { File::removeTokenAsync('DB_IN_IMAGE'); } + + // DB_PUSH wraps the nightly push of the sanitized dump, which only a + // dump store source accepts. + if (!in_array($v, [self::S3, self::GDRIVE], TRUE)) { + File::removeTokenAsync('DB_PUSH'); + } } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index fa9f490391..abc1fefba6 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -21,6 +21,8 @@ class MigrationFetchSource extends AbstractHandler { const S3 = 's3'; + const GDRIVE = 'gdrive'; + /** * {@inheritdoc} */ @@ -53,6 +55,7 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', + self::GDRIVE => 'Google Drive', ]; if (isset($responses[HostingProvider::id()])) { @@ -133,6 +136,7 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, + self::GDRIVE, ]; foreach ($types as $type) { diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php index 9ca89d1241..0b9cd9d62b 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php @@ -36,6 +36,9 @@ public static function dataProviderHandlerProcess(): \Iterator { yield 'db_fetch_source_s3' => [ static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::S3), ]; + yield 'db_fetch_source_gdrive' => [ + static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::GDRIVE), + ]; } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index 0c63da7a4c..e410c67712 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -77,6 +77,20 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); }), ]; + yield 'migration_fetch_source_gdrive' => [ + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->prompts[Migration::id()] = TRUE; + $test->prompts[MigrationFetchSource::id()] = MigrationFetchSource::GDRIVE; + }), + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_SOURCE=gdrive'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); + // The migration database is fetched from the store but never pushed. + $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'vortex-push-db'); + }), + ]; yield 'migration_fetch_source_container_registry' => [ static::cw(function (AbstractHandlerProcessTestCase $test): void { $test->prompts[Migration::id()] = TRUE; diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats index 863221bf25..987005923e 100644 --- a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats +++ b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats @@ -88,6 +88,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success @@ -117,6 +118,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success @@ -143,6 +145,7 @@ fixture_gdrive_key() { mock_curl=$(mock_command "curl") mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 + mock_set_status "${mock_curl}" 0 3 run .vortex/tooling/src/vortex-fetch-db-gdrive assert_success diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index f81a93af38..338b9a7e8a 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -386,6 +386,10 @@ load ../_helper.bash mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") + mock_set_output "${mock_docker}" "Started the stack." 1 + mock_set_output "${mock_docker}" "Created the directory." 2 + mock_set_output "${mock_docker}" "Copied the dump file." 3 + mock_set_output "${mock_docker}" "Provisioned the site." 4 mock_set_status "${mock_docker}" 1 5 run .vortex/tooling/src/vortex-refresh-db From 7fe9d34db202826ab3265cc48ae11a160128aef8 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:31:49 +1100 Subject: [PATCH 03/40] [#154] Documented pushing the sanitized database to a Google Drive or S3 dump store. --- .circleci/vortex-test-common.yml | 12 ++++ .github/workflows/vortex-test-didi.yml | 44 +++++++++++++ .../content/continuous-integration/README.mdx | 53 ++++++++++++++- .../docs/content/development/provision.mdx | 5 ++ .../docs/content/development/variables.mdx | 17 ++++- .vortex/docs/content/features.mdx | 9 ++- .vortex/docs/cspell.json | 2 + .../tooling/playground/try-fetch-db-gdrive.sh | 57 ++++++++++++++++ .../tooling/playground/try-push-db-gdrive.sh | 65 +++++++++++++++++++ 9 files changed, 257 insertions(+), 7 deletions(-) create mode 100755 .vortex/tooling/playground/try-fetch-db-gdrive.sh create mode 100755 .vortex/tooling/playground/try-push-db-gdrive.sh diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index ad1d11660a..5d52519f05 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -183,6 +183,18 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 0aaf14be63..17399bfe49 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -64,6 +64,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -129,6 +136,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -191,6 +202,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order @@ -586,6 +608,13 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii + #;< DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} + VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + #;> DB_FETCH_SOURCE_S3 + #;< DB_FETCH_SOURCE_GDRIVE + VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + #;> DB_FETCH_SOURCE_GDRIVE VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -650,6 +679,10 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE + #;< DB_PUSH + # Refresh the database dump in the database fetch source. + echo "VORTEX_PUSH_DB_PROCEED=1" + #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -731,6 +764,17 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY + #;< DB_PUSH + # Push the freshly-exported sanitized database dump to the database fetch + # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the + # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index 20ded51063..e6404015a4 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,6 +79,9 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry +- On the nightly run of a project with the `s3` or `gdrive` fetch source, + pushes a sanitized dump to that source (see [Push the sanitized database + to a dump store](#push-the-sanitized-database-to-a-dump-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly @@ -386,7 +389,7 @@ future **Vortex** update ships. The scheduled run builds a database container image from a dump file. With the `container_registry` fetch source, set `VORTEX_REFRESH_DB_SOURCE` to the -source of that file: `url`, `ftp`, `acquia`, `lagoon` or `s3`. +source of that file: `url`, `ftp`, `acquia`, `lagoon`, `s3` or `gdrive`. `VORTEX_REFRESH_DB_SOURCE` selects a fetch script, and each fetch script reads its own variables. The installer adds to `.env` the variables of the @@ -409,6 +412,54 @@ For the migration database, set `VORTEX_REFRESH_DB2_SOURCE` and the variables apply to both databases. See [Refresh the migration database image](../development/migrations.mdx#refresh-the-migration-database-image). +### Push the sanitized database to a dump store + +A dump store is an `s3` or `gdrive` fetch source that holds a sanitized +copy of your hosting database. Developers and branch builds fetch that +copy and need no hosting access. On the nightly run, the `database` job +fetches a dump from `VORTEX_REFRESH_DB_SOURCE` and sanitizes it. Its +**Push DB** step then uploads the dump to the fetch source. + +To refresh a dump store every night: + +1. In `.env`, set `VORTEX_FETCH_DB_SOURCE` to `s3` or `gdrive`. The + installer sets it and adds the variables of that source. +2. In `.env`, set `VORTEX_REFRESH_DB_SOURCE` to the hosting source: `url`, + `ftp`, `acquia` or `lagoon`. Add the variables of that source from the + table in [Dump file source for a database + image](#dump-file-source-for-a-database-image). +3. Add the credentials of the dump store as CI secrets: + + | Source | Secrets | + |---|---| + | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | + | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file | + + GitHub Actions passes these secrets to the `database` job. Add the + secrets of the hosting source to the `env` block of that job. + +4. After the next nightly run, confirm that its **Push DB** step ends with + `Finished database push.` + +The `s3` push writes to the bucket, region and prefix that the fetch +reads. The `gdrive` push replaces a file with the same name in place, so +the file keeps its ID and sharing. Both pushes stop with an error when +`VORTEX_PROVISION_SANITIZE_DB_SKIP` is `1`. + +The `gdrive` source signs in with a Google Cloud service account key. +Service accounts have no storage of their own, so the folder must be in a +shared drive. To set up the folder: + +1. In the Google Cloud console, enable the Google Drive API and create a + service account with a JSON key. +2. In a shared drive, create a folder and add the service account email + as a **Content manager**. +3. Copy the folder ID, the last segment of the folder URL, into + `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. +4. Give developers the key of a second service account with the + **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in + `.env.local` to the path of the key file or to its content. + ## Trigger conditions Both providers build on branch pushes, pull requests and tag pushes, plus diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index ec35e263e2..eecfbc5c06 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -375,10 +375,15 @@ following values: - `container_registry`: a database container image, pulled instead of a file - `s3`: a file in an S3 bucket +- `gdrive`: a file in a Google Drive folder - `none`: no source; you put the dump into `.data` yourself Any other value stops the command with an error that lists these values. +The nightly CI run can refresh an `s3` or `gdrive` source with a sanitized +dump from your hosting. See [Push the sanitized database to a dump +store](../continuous-integration/README.mdx#push-the-sanitized-database-to-a-dump-store). + Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the environment named in `VORTEX_FETCH_DB_ENVIRONMENT`. `.env` ships with diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index be78299b73..cb724c71b3 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -323,6 +323,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_FTP_FILE` | Second database dump FTP file name. | `db2.sql` | `.env` | | `VORTEX_FETCH_DB2_FTP_HOST` | Second database dump FTP host. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_PORT` | Second database dump FTP port. | `21` | `.env` | +| `VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID` | Google Drive folder ID with the second database dump file. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_BUCKET` | AWS S3 bucket name for second database fetch. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env` | | `VORTEX_FETCH_DB2_SOURCE` | Second database fetch source. | `url` | `.env` | @@ -352,6 +353,10 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_FTP_PASS` | Database dump FTP password. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_PORT` | Database dump FTP port. | `21` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_USER` | Database dump FTP user. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | +| `VORTEX_FETCH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID with the database dump file.

The ID is the last segment of the folder URL. The folder must be in a shared drive and shared with the service account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-gdrive` | +| `VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | | `VORTEX_FETCH_DB_LAGOON_DB_DIR` | Directory where DB dumps are stored on the host. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_DB_FILE` | Database dump file name on the host. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_PROJECT` | Lagoon project name. | `${LAGOON_PROJECT}` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | @@ -369,7 +374,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env`, `.vortex/tooling/src/vortex-fetch-db-s3` | | `VORTEX_FETCH_DB_S3_SECRET_KEY` | AWS secret key for S3 database fetch. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-s3` | -| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db` | +| `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-push-db` | | `VORTEX_FETCH_DB_SSH_FILE` | Default SSH file used if custom fingerprint is not provided. | `${HOME}/.ssh/id_rsa` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_UNZIP_PASSWORD` | Password for unzipping password-protected zip files. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-url` | @@ -394,6 +399,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | +| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -549,17 +555,22 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | +| `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Fetch source of the database dump file to import into the base image.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream fetch source of the database dump file.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE.

Also required when the fetch source is a dump store ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED is set to `1`: the dump pushed to the store comes from an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index 70df8ab568..44c2adb358 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -169,7 +169,8 @@ import { - _Lint_ the code without provisioning a site - _Fetch the database_ from the hosting provider and cache it for the - day, with a nightly refresh of the database container image + day, with a nightly refresh of the database container image or of a + sanitized dump in S3 or Google Drive - _Build_ and test a fully provisioned site, with [Behat](https://docs.behat.org/en/latest/) tests split across parallel containers @@ -244,7 +245,8 @@ import { ### Other hosting providers - Webhook deployments for any provider that accepts a deployment URL - - Database fetch from a URL, FTP, S3 or a container registry + - Database fetch from a URL, FTP, S3, Google Drive or a container + registry :::note @@ -383,7 +385,8 @@ import { [`drevops/vortex-tooling`](https://packagist.org/packages/drevops/vortex-tooling) Composer package, run the same way in every environment - Customization through environment variables - - Database sourced from the hosting provider, a URL, FTP, S3 or a + - Database sourced from the hosting provider, a URL, FTP, S3, Google + Drive or a container image seeded with [Database data images](https://github.com/drevops/database-data-images) diff --git a/.vortex/docs/cspell.json b/.vortex/docs/cspell.json index 6b66040eb6..5d4b399675 100644 --- a/.vortex/docs/cspell.json +++ b/.vortex/docs/cspell.json @@ -13,6 +13,7 @@ "Buildx", "Diffy", "Dockerfiles", + "GDRIVE", "GHSA", "Incapsula", "MAILHUB", @@ -53,6 +54,7 @@ "drush", "ergebnis", "fieldsets", + "gdrive", "gherkinlint", "gitleaks", "hadolint", diff --git a/.vortex/tooling/playground/try-fetch-db-gdrive.sh b/.vortex/tooling/playground/try-fetch-db-gdrive.sh new file mode 100755 index 0000000000..43fd135f42 --- /dev/null +++ b/.vortex/tooling/playground/try-fetch-db-gdrive.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +## +# Manual test script for Google Drive database fetch. +# +# Fetches a database dump file from a Google Drive folder using +# fetch-db-gdrive. +# +# Usage: +# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" +# export GDRIVE_FOLDER_ID="your-folder-id" +# ./try-fetch-db-gdrive.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Google Drive credentials and settings. +GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" +GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" + +# Validate required variables. +if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then + echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" + echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" + exit 1 +fi + +if [ -z "${GDRIVE_FOLDER_ID}" ]; then + echo "Error: GDRIVE_FOLDER_ID environment variable is required" + echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" + exit 1 +fi + +echo "Testing Google Drive database fetch..." +echo "" +echo "Folder ID: ${GDRIVE_FOLDER_ID}" +echo "DB file: ${VORTEX_DB_FILE:-db.sql}" +echo "Local dir: ${VORTEX_DB_DIR:-./.data}" +echo "" + +cd "${PROJECT_ROOT}" || exit 1 + +export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" +export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" + +./vendor/drevops/vortex-tooling/src/vortex-fetch-db-gdrive + +echo "" +echo "Fetch complete!" diff --git a/.vortex/tooling/playground/try-push-db-gdrive.sh b/.vortex/tooling/playground/try-push-db-gdrive.sh new file mode 100755 index 0000000000..9cddbab46f --- /dev/null +++ b/.vortex/tooling/playground/try-push-db-gdrive.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +## +# Manual test script for Google Drive database push. +# +# Uploads a database dump file to a Google Drive folder in a shared drive +# using push-db-gdrive. Run it twice to check that the second run replaces +# the file instead of creating a duplicate. +# +# Usage: +# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" +# export GDRIVE_FOLDER_ID="your-folder-id" +# ./try-push-db-gdrive.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Google Drive credentials and settings. +GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" +GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" + +# Validate required variables. +if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then + echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" + echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" + exit 1 +fi + +if [ -z "${GDRIVE_FOLDER_ID}" ]; then + echo "Error: GDRIVE_FOLDER_ID environment variable is required" + echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" + exit 1 +fi + +cd "${PROJECT_ROOT}" || exit 1 + +# Create a test dump file when none exists. +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" +if [ ! -f "${db_dir}/${db_file}" ]; then + mkdir -p "${db_dir}" + echo "-- Test dump created by try-push-db-gdrive.sh on $(date -u)" >"${db_dir}/${db_file}" +fi + +echo "Testing Google Drive database push..." +echo "" +echo "Folder ID: ${GDRIVE_FOLDER_ID}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" +export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" + +./vendor/drevops/vortex-tooling/src/vortex-push-db-gdrive + +echo "" +echo "Push complete!" From 64fd7573a9c815c0df8ae85d373281d312542d49 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:33:45 +1100 Subject: [PATCH 04/40] Updated snapshots. --- .../db_fetch_source_container_registry/.env | 6 +- .../db_fetch_source_gdrive/.env | 45 +++++ .../db_fetch_source_gdrive/.env.local.example | 10 + .../.github/workflows/build-test-deploy.yml | 33 ++++ .../handler_process/db_fetch_source_s3/.env | 10 +- .../.github/workflows/build-test-deploy.yml | 34 ++++ .../.env | 6 +- .../migration_fetch_source_gdrive/.ahoy.yml | 51 +++++ .../migration_fetch_source_gdrive/.env | 30 +++ .../.github/workflows/build-test-deploy.yml | 22 +++ .../composer.json | 9 + .../docker-compose.yml | 52 +++++ .../scripts/provision-20-migration.sh | 187 ++++++++++++++++++ .../phpunit/Drupal/DatabaseSettingsTest.php | 110 +++++++++++ .../Drupal/EnvironmentSettingsTest.php | 14 ++ .../tests/phpunit/Drupal/SettingsTestCase.php | 8 + .../migrations/ys_migrate_categories.yml | 29 +++ .../custom/ys_migrate/ys_migrate.info.yml | 9 + .../web/sites/default/settings.migration.php | 21 ++ .../web/sites/default/settings.php | 15 ++ 20 files changed, 693 insertions(+), 8 deletions(-) create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml create mode 100755 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index f98032044d..e6357bf6c3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -14,9 +14,9 @@ +VORTEX_DB_IMAGE=the_empire/star_wars:latest +# Source of the database dump file the nightly CI job imports into a clean -+# base image to build the database container image: url, ftp, acquia, lagoon -+# or s3. The fetch script of that source reads its own variables, such as -+# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. ++# base image to build the database container image: url, ftp, acquia, lagoon, ++# s3 or gdrive. The fetch script of that source reads its own variables, such ++# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env new file mode 100644 index 0000000000..eb9fa4209c --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env @@ -0,0 +1,45 @@ +@@ -180,13 +180,22 @@ + VORTEX_DB_FILE=db.sql + + # Database fetch source. +-VORTEX_FETCH_DB_SOURCE=url ++VORTEX_FETCH_DB_SOURCE=gdrive + +-# Database dump file sourced from a URL. ++# Database dump file sourced from Google Drive. ++ ++# Google Drive folder ID with the database dump file. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# The ID is the last segment of the folder URL. The folder must be in a shared ++# drive and shared with the service account. ++VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= + ++# Source of the database dump file the nightly CI job sanitizes and pushes to ++# the database fetch source: url, ftp, acquia or lagoon. The fetch script of ++# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. ++# Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url ++ + # Environment to fetch the database from. + # + # Applies to hosting environments. +@@ -254,17 +263,3 @@ + # with optional names in the format "email|name". + # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" + VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" +- +-################################################################################ +-# DEMO # +-################################################################################ +-# # +-# Override project-specific values for demonstration purposes. # +-# Used to showcase Vortex without asking users to perform additional steps. # +-# # +-# Remove this section after completing database fetch integration. # +-# # +-################################################################################ +- +-# URL of the database used for demonstration with URL database fetch type. +-VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example new file mode 100644 index 0000000000..e62dbc8c32 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example @@ -0,0 +1,10 @@ +@@ -32,3 +32,9 @@ + + # Always override existing fetched DB dump. + VORTEX_FETCH_DB_FORCE=1 ++ ++# Database dump file sourced from Google Drive. ++# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key ++ ++# Google Cloud service account key: the JSON content or a path to the JSON key file. ++VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..88c93392b7 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml @@ -0,0 +1,33 @@ +@@ -191,6 +191,7 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -241,6 +242,8 @@ + { + echo "VORTEX_CI_DB_CACHE_FALLBACK=no" + echo "VORTEX_FRONTEND_BUILD_SKIP=1" ++ # Refresh the database dump in the database fetch source. ++ echo "VORTEX_PUSH_DB_PROCEED=1" + } >> "$GITHUB_ENV" + + - name: Create cache keys files for database caching +@@ -274,6 +277,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 5d9d274041..b26eb7cb6a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,16 @@ +@@ -180,13 +180,22 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -16,11 +16,17 @@ + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 ++ ++# Source of the database dump file the nightly CI job sanitizes and pushes to ++# the database fetch source: url, ftp, acquia or lagoon. The fetch script of ++# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. ++# Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. # # Applies to hosting environments. -@@ -254,17 +257,3 @@ +@@ -254,17 +263,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..d8ec3cb540 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -0,0 +1,34 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -241,6 +243,8 @@ + { + echo "VORTEX_CI_DB_CACHE_FALLBACK=no" + echo "VORTEX_FRONTEND_BUILD_SKIP=1" ++ # Refresh the database dump in the database fetch source. ++ echo "VORTEX_PUSH_DB_PROCEED=1" + } >> "$GITHUB_ENV" + + - name: Create cache keys files for database caching +@@ -274,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index cdfd843852..fc0db2960c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -24,9 +24,9 @@ + +# Source of the second database dump file the nightly CI job imports into a +# clean base image to rebuild the migration database container image: url, -+# ftp, acquia, lagoon or s3. The fetch script of that source reads its own -+# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets -+# in CI. ++# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its ++# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml new file mode 100644 index 0000000000..11f7a073e6 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml @@ -0,0 +1,51 @@ +@@ -32,6 +32,7 @@ + cmd: | + export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-${PWD##*/}} + export VORTEX_HOST_DB_PORT=$(docker compose port database 3306 2>/dev/null | cut -d : -f 2) ++ export VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SOLR_PORT=$(docker compose port solr 8983 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SELENIUM_VNC_PORT=$(docker compose port chrome 7900 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_HAS_SEQUELACE=$(uname -a | grep -i -q darwin && mdfind -name 'Sequel Ace' 2>/dev/null | grep -q "Ace" && echo 1 || true) +@@ -45,6 +46,14 @@ + open "mysql://${DATABASE_USERNAME:-drupal}:${DATABASE_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB_PORT}/drupal" -a "Sequel Ace" \ + || echo "Not a supported OS or Sequel Ace is not installed." + ++ db2: ++ usage: Open DB2 in Sequel Ace. ++ cmd: | ++ uname -a | grep -i -q darwin && test -d "${VORTEX_HOST_SEQUELACE_PATH:-/Applications/Sequel Ace.app}" && \ ++ VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) && \ ++ open "mysql://${DATABASE2_USERNAME:-drupal}:${DATABASE2_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB2_PORT}/drupal" -a "Sequel Ace" \ ++ || echo "Not a supported OS or Sequel Ace is not installed." ++ + # ---------------------------------------------------------------------------- + # Container commands. + # ---------------------------------------------------------------------------- +@@ -122,6 +131,13 @@ + case " $* " in *" --fresh "*) export VORTEX_FETCH_DB_FRESH=1 VORTEX_FETCH_DB_FORCE=1;; esac + ./vendor/bin/vortex-fetch-db + ++ fetch-db2: ++ usage: Fetch second database (migration). Run with "--fresh" to force. ++ aliases: [download-db2, db2-download, db2-fetch] ++ cmd: | ++ case " $* " in *" --fresh "*) export VORTEX_FETCH_DB2_FRESH=1 VORTEX_FETCH_DB2_FORCE=1;; esac ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + fetch-files: + usage: Fetch public files. + aliases: [download-files, files-download, files-fetch] +@@ -134,6 +150,13 @@ + docker compose rm --force --stop --volumes database + docker compose build database + ahoy up ++ ++ reload-db2: ++ usage: Reload the migration database container using local database image. ++ cmd: | ++ ahoy confirm "Running this command will replace your current database. Are you sure?" || exit 0 ++ docker compose rm --force --stop --volumes database2 ++ ahoy up -- --build database2 + + provision: + usage: Provision a site from the database dump or profile. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env new file mode 100644 index 0000000000..bce43c946e --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env @@ -0,0 +1,30 @@ +@@ -195,6 +195,29 @@ + VORTEX_FETCH_DB_ENVIRONMENT=prod + + ################################################################################ ++# SECOND DATABASE (MIGRATION) # ++################################################################################ ++ ++# Second database for migrations. ++# See settings.migration.php for database credentials. ++ ++# Second database dump file name. ++VORTEX_FETCH_DB2_FILE=db2.sql ++ ++# Second database fetch source. ++VORTEX_FETCH_DB2_SOURCE=gdrive ++ ++# Second database dump file sourced from Google Drive. ++ ++# Google Drive folder ID with the second database dump file. ++VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= ++ ++# Environment to fetch the second database from. ++# ++# Applies to hosting environments. ++VORTEX_FETCH_DB2_ENVIRONMENT=prod ++ ++################################################################################ + # FILES SOURCE # + ################################################################################ + diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..4292b4f549 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml @@ -0,0 +1,22 @@ +@@ -270,6 +270,9 @@ + echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" + timeout-minutes: 30 + ++ - name: Fetch migration DB ++ run: VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + - name: Export DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi +@@ -431,6 +434,11 @@ + docker compose exec cli mkdir -p .data + docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql + rm -f .data/db.sql ++ fi ++ if [ -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ]; then ++ docker compose exec -T cli mkdir -p .data ++ docker compose --progress quiet cp -L ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" cli:"/app/.data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ++ rm -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" + fi + docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision + timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json new file mode 100644 index 0000000000..b90a23c71c --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json @@ -0,0 +1,9 @@ +@@ -26,6 +26,8 @@ + "drupal/linkit": "__VERSION__", + "drupal/menu_trail_by_path": "__VERSION__", + "drupal/metatag": "__VERSION__", ++ "drupal/migrate_plus": "__VERSION__", ++ "drupal/migrate_tools": "__VERSION__", + "drupal/navigation_extra_tools": "__VERSION__", + "drupal/pathauto": "__VERSION__", + "drupal/redirect": "__VERSION__", diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml new file mode 100644 index 0000000000..70ac2fcbc8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml @@ -0,0 +1,52 @@ +@@ -57,6 +57,16 @@ + DRUPAL_SHIELD_PASS: ${DRUPAL_SHIELD_PASS:-} + # Redis integration flag. + DRUPAL_REDIS_ENABLED: ${DRUPAL_REDIS_ENABLED:-} ++ # Local migration database host (not used in production). ++ DATABASE2_HOST: database2 ++ # Local migration database name (not used in production). ++ DATABASE2_NAME: drupal ++ # Local migration database user (not used in production). ++ DATABASE2_USERNAME: drupal ++ # Local migration database password (not used in production). ++ DATABASE2_PASSWORD: drupal ++ # Local migration database port (not used in production). ++ DATABASE2_PORT: 3306 + + # The default user under which the containers should run. + x-user: &default-user +@@ -155,6 +165,20 @@ + ports: + - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database 3306`. + ++ database2: ++ image: "${VORTEX_DB2_IMAGE:-uselagoon/mysql-8.4:__VERSION__}" # Use custom database image (if defined) or fallback to standard database image. ++ environment: ++ <<: *default-environment ++ MYSQL_DATABASE: drupal ++ MYSQL_USER: drupal ++ MYSQL_PASSWORD: drupal ++ <<: *default-user ++ # dclint disable-line no-duplicate-exported-ports # container-only port, published on a random host port ++ ports: ++ - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database2 3306`. ++ labels: ++ lagoon.type: mariadb # See https://docs.lagoon.sh/using-lagoon-advanced/service-types/ ++ + redis: + image: uselagoon/valkey-8:__VERSION__ + +@@ -198,11 +222,13 @@ + depends_on: + - clamav + - database ++ - database2 + - redis + - solr + command: + - clamav:3310 + - database:3306 ++ - database2:3306 + - redis:6379 + - solr:8983 + - curl -s "http://solr:8983/solr/drupal/select?q=*:*&rows=0&wt=json" | grep response diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh new file mode 100755 index 0000000000..3000594760 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +## +# Import migration database and run migrations. +# +# This script is called during site provisioning via provision.sh. +# Customize the migration names at the bottom of this file. +# +# shellcheck disable=SC2086 + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Skip all migrations. +DRUPAL_MIGRATION_SKIP="${DRUPAL_MIGRATION_SKIP:-0}" + +# Skip rollback of migrations before import. +DRUPAL_MIGRATION_ROLLBACK_SKIP="${DRUPAL_MIGRATION_ROLLBACK_SKIP:-1}" + +# Limit the number of entities to import. Set to 'all' to import all. +DRUPAL_MIGRATION_IMPORT_LIMIT="${DRUPAL_MIGRATION_IMPORT_LIMIT:-50}" + +# Update already imported entities during migration. +DRUPAL_MIGRATION_UPDATE="${DRUPAL_MIGRATION_UPDATE:-0}" + +# Feedback frequency for migration progress. +DRUPAL_MIGRATION_FEEDBACK="${DRUPAL_MIGRATION_FEEDBACK:-50}" + +# Import migration source database. Set to 1 to import, 0 to skip. +DRUPAL_MIGRATION_SOURCE_DB_IMPORT="${DRUPAL_MIGRATION_SOURCE_DB_IMPORT:-${VORTEX_PROVISION_OVERRIDE_DB:-0}}" + +# Table name to probe in the source database to verify it is not corrupted. +DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE="${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE:-categories}" + +# Disable the search server while migrations run. Set to 0 to keep indexing. +DRUPAL_MIGRATION_SEARCH_DISABLE="${DRUPAL_MIGRATION_SEARCH_DISABLE:-1}" + +# Directory with database dump file. +VORTEX_DB_DIR="${VORTEX_DB_DIR:-./.data}" + +# Migration database dump file name. +VORTEX_FETCH_DB2_FILE="${VORTEX_FETCH_DB2_FILE:-db2.sql}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { printf " ==> %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { printf " > %s\n" "${1}"; } +pass() { printf " + %s\n" "${1}"; } +fail() { printf " ! %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +drush() { ./vendor/bin/drush -y "$@"; } + +# ------------------------------------------------------------------------------ + +info "Started migration operations." + +environment="$(drush php:eval "print \Drupal\Core\Site\Settings::get('environment');")" +note "Environment: ${environment}" + +# Skip migrations in production. +if [ "${environment}" = "prod" ]; then + DRUPAL_MIGRATION_SKIP=1 +fi + +note "Migration skip: ${DRUPAL_MIGRATION_SKIP}" +note "Migration limit: ${DRUPAL_MIGRATION_IMPORT_LIMIT}" +note "Migration skip rollback: ${DRUPAL_MIGRATION_ROLLBACK_SKIP}" +note "Migration update: ${DRUPAL_MIGRATION_UPDATE}" +note "Migration feedback: ${DRUPAL_MIGRATION_FEEDBACK}" +note "Source DB import: ${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" +note "Search disable: ${DRUPAL_MIGRATION_SEARCH_DISABLE}" +echo + +if [ "${DRUPAL_MIGRATION_SKIP}" = "1" ]; then + info "Skipped migrations. DRUPAL_MIGRATION_SKIP is set to 1." + exit 0 +fi + +# Helper function to run a single migration with configured options. +run_migration() { + local migration_name="${1}" + shift + + drush migrate:reset-status "${migration_name}" || fail "Failed to reset migration status for ${migration_name}." + + task "Running migration: ${migration_name}." + local opts=() + + opts+=("--feedback=${DRUPAL_MIGRATION_FEEDBACK}") + + if printf '%s\n' "${DRUPAL_MIGRATION_IMPORT_LIMIT}" | grep -q '^[0-9][0-9]*$' && [ "${DRUPAL_MIGRATION_IMPORT_LIMIT}" -gt 0 ]; then + opts+=("--limit=${DRUPAL_MIGRATION_IMPORT_LIMIT}") + fi + + if [ "${DRUPAL_MIGRATION_UPDATE}" = "1" ]; then + opts+=("--update") + fi + + # Add any additional arguments passed to the function. + opts+=("$@") + + drush migrate:import "${opts[@]}" "${migration_name}" || { + drush migrate:messages "${migration_name}" + fail "Failed to run migration ${migration_name}." + } + + pass "Migrated: ${migration_name}." +} + +# Detect if existing migration source database is corrupted. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" != "1" ]; then + note "Source database import is set to be skipped. Checking existing database." + task "Probing for '${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}' table in the source database." + if drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + pass "Source database is intact." + else + note "Migration source database is corrupted or empty. Re-importing." + DRUPAL_MIGRATION_SOURCE_DB_IMPORT=1 + pass "Marked the source database for re-import." + fi +fi + +# Import the migration source database from the dump file. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" = "1" ]; then + task "Importing migration source database." + + [ ! -f "${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" ] && fail "Migration source database file not found. Please run 'ahoy fetch-db2'." + + drush sql:drop --database=migrate + # shellcheck disable=SC2091 + $(drush sql:connect --database=migrate) <"${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" + + pass "Imported migration source database." +else + note "Using existing migration source database." +fi + +task "Verifying migration source database." +if ! drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + drush sql:query --database=migrate "SHOW TABLES;" + fail "Migration source database is corrupted." +fi +pass "Verified migration source database." + +# Enable custom migration modules. +task "Enabling migration modules." +drush pm:install ys_migrate +pass "Enabled migration modules." + +info "Starting migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Disabling Search API Solr server." + drush search-api:server-disable solr || true + pass "Disabled Search API Solr server." +fi + +if [ "${DRUPAL_MIGRATION_ROLLBACK_SKIP}" = "1" ]; then + note "Skipped rollback of all migrations." +else + task "Rolling back all migrations." + drush migrate:rollback --all || true + pass "Rolled back all migrations." +fi +echo + +# ------------------------------------------------------------------------------ +# Add your migrations below. +# ------------------------------------------------------------------------------ + +run_migration ys_migrate_categories + +echo +note "Finished migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Enabling Search API Solr server." + drush search-api:server-enable solr || true + drush search-api:enable-all || true + pass "Enabled Search API Solr server." +fi + +drush migrate:status + +info "Finished migration operations." diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php new file mode 100644 index 0000000000..956cd535d4 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php @@ -0,0 +1,110 @@ +@@ -47,6 +47,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -74,6 +85,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -101,6 +123,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -128,9 +161,55 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + ++ yield [ ++ [ ++ 'DATABASE2_NAME' => 'migrate_db_name', ++ 'DATABASE2_USERNAME' => 'migrate_db_user', ++ 'DATABASE2_PASSWORD' => 'migrate_db_pass', ++ 'DATABASE2_HOST' => 'migrate_db_host', ++ 'DATABASE2_PORT' => '3307', ++ ], ++ [ ++ 'default' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '3306', ++ 'charset' => 'utf8mb4', ++ 'collation' => 'utf8mb4_general_ci', ++ 'driver' => 'mysql', ++ 'prefix' => '', ++ ], ++ ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'migrate_db_name', ++ 'username' => 'migrate_db_user', ++ 'password' => 'migrate_db_pass', ++ 'host' => 'migrate_db_host', ++ 'port' => '3307', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], ++ ], ++ ]; + } + + } diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php new file mode 100644 index 0000000000..8adb9246e8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php @@ -0,0 +1,14 @@ +@@ -163,6 +163,13 @@ + $databases['default']['default']['collation'] = 'utf8_general_ci'; + $databases['default']['default']['driver'] = 'mysql'; + $databases['default']['default']['prefix'] = ''; ++ $databases['migrate']['default']['database'] = 'drupal'; ++ $databases['migrate']['default']['username'] = 'drupal'; ++ $databases['migrate']['default']['password'] = 'drupal'; ++ $databases['migrate']['default']['host'] = 'localhost'; ++ $databases['migrate']['default']['port'] = ''; ++ $databases['migrate']['default']['prefix'] = ''; ++ $databases['migrate']['default']['driver'] = 'mysql'; + $this->assertEquals($databases, $this->databases); + + // Verify key config overrides. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php new file mode 100644 index 0000000000..350cc85c04 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php @@ -0,0 +1,8 @@ +@@ -62,6 +62,7 @@ + const ALLOWED_ENV_VARS = [ + // Service variables. + 'DATABASE_', ++ 'DATABASE2_', + 'REDIS_', + 'CLAMAV_', + 'COMPOSE_', diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml new file mode 100644 index 0000000000..0494698e5d --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml @@ -0,0 +1,29 @@ +id: ys_migrate_categories +label: 'Migrate categories to tags' +migration_tags: + - ys_migrate + +source: + plugin: table + key: migrate + table_name: categories + id_fields: + id: + type: integer + fields: + id: id + name: name + description: description + +process: + name: name + description/value: description + description/format: + plugin: default_value + default_value: plain_text + vid: + plugin: default_value + default_value: tags + +destination: + plugin: 'entity:taxonomy_term' diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml new file mode 100644 index 0000000000..96c67179ac --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml @@ -0,0 +1,9 @@ +name: star wars Migrate +type: module +description: Migrations for star wars site. +core_version_requirement: ^11 +package: star_wars +dependencies: + - drupal:migrate + - migrate_plus:migrate_plus + - migrate_tools:migrate_tools diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php new file mode 100644 index 0000000000..660869b39f --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php @@ -0,0 +1,21 @@ + (getenv('DATABASE2_NAME') ?: getenv('DATABASE2_DATABASE')) ?: 'drupal', + 'username' => getenv('DATABASE2_USERNAME') ?: 'drupal', + 'password' => getenv('DATABASE2_PASSWORD') ?: 'drupal', + 'host' => getenv('DATABASE2_HOST') ?: 'localhost', + 'port' => getenv('DATABASE2_PORT') ?: '', + 'prefix' => '', + 'driver' => 'mysql', +]; diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php new file mode 100644 index 0000000000..c04331d75e --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php @@ -0,0 +1,15 @@ +@@ -167,6 +167,14 @@ + } + + //////////////////////////////////////////////////////////////////////////////// ++/// MIGRATION SETTINGS /// ++//////////////////////////////////////////////////////////////////////////////// ++// @todo Migration. Remove when migration configuration is no longer required. ++if (file_exists($app_root . '/' . $site_path . '/settings.migration.php')) { ++ require $app_root . '/' . $site_path . '/settings.migration.php'; ++} ++ ++//////////////////////////////////////////////////////////////////////////////// + /// LOCAL OVERRIDE /// + //////////////////////////////////////////////////////////////////////////////// + // @see https://www.vortextemplate.com/docs/development/settings#local-overrides From 86d1b974c7a31763278b099282dfe5e2cffe6a09 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:39:02 +1100 Subject: [PATCH 05/40] [#154] Fetched the dump from the upstream source in 'vortex-fetch-db' so the first push to an empty dump store succeeds. --- .env.local.example | 2 +- .../content/continuous-integration/README.mdx | 4 +- .vortex/tooling/src/vortex-fetch-db | 15 +++ .vortex/tooling/src/vortex-refresh-db | 28 ++---- .vortex/tooling/tests/unit/fetch-db.bats | 85 +++++++++++++++++ .vortex/tooling/tests/unit/refresh-db.bats | 94 ++++--------------- 6 files changed, 126 insertions(+), 102 deletions(-) diff --git a/.env.local.example b/.env.local.example index e876e08dd8..4f87f00489 100644 --- a/.env.local.example +++ b/.env.local.example @@ -73,7 +73,7 @@ VORTEX_FETCH_DB_S3_SECRET_KEY= # Database dump file sourced from Google Drive. # Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key -# Google Cloud service account key: the JSON content or a path to the JSON key file. +# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= #;> DB_FETCH_SOURCE_GDRIVE diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index e6404015a4..dcd492e4f3 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -433,7 +433,7 @@ To refresh a dump store every night: | Source | Secrets | |---|---| | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | - | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file | + | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file on 1 line, such as the output of `jq -c . key.json` | GitHub Actions passes these secrets to the `database` job. Add the secrets of the hosting source to the `env` block of that job. @@ -458,7 +458,7 @@ shared drive. To set up the folder: `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. 4. Give developers the key of a second service account with the **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in - `.env.local` to the path of the key file or to its content. + `.env.local` to the path of the key file. ## Trigger conditions diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index faf32b4ac9..f1abf35058 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -55,6 +55,21 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 +# A dump store ('gdrive' or 's3') receives the dump that the nightly refresh +# pushes, so with a push requested the dump comes from its upstream source. +# The store may hold no dump yet on the first push. +if [ -z "${_db_index}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + case "${VORTEX_FETCH_DB_SOURCE}" in + gdrive | s3) + case "${VORTEX_REFRESH_DB_SOURCE:-}" in + "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE:-}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + esac + note "Fetching from the ${VORTEX_REFRESH_DB_SOURCE} source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + VORTEX_FETCH_DB_SOURCE="${VORTEX_REFRESH_DB_SOURCE}" + ;; + esac +fi + # The fetch script is resolved before the existing dump check, so an # unsupported source fails even when a cached dump would be reused. case "${VORTEX_FETCH_DB_SOURCE}" in diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index 9b2f01e2e5..df899cafd5 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -16,9 +16,8 @@ # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # # With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED -# set to 1, the dump file is fetched from VORTEX_REFRESH_DB_SOURCE before -# provisioning and the database is sanitized after it, so the exported dump -# can be pushed to the store with 'vortex-push-db'. +# set to 1, the database is sanitized after provisioning, so the exported +# dump can be pushed to the store with 'vortex-push-db'. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -52,8 +51,8 @@ VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" # indexed database uses VORTEX_REFRESH_DB2_SOURCE. # # Also required when the fetch source is a dump store ('gdrive' or 's3') and -# VORTEX_PUSH_DB_PROCEED is set to 1: the dump pushed to the store comes from -# an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. +# VORTEX_PUSH_DB_PROCEED is set to 1: 'vortex-fetch-db' then fetches the dump +# from this source, such as 'url', 'ftp', 'acquia' or 'lagoon'. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -108,7 +107,7 @@ rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 # A dump store ('gdrive' or 's3') receives the dump this refresh exports, so -# with a push requested the dump is fetched again from its upstream source. +# with a push requested the exported dump must be sanitized. refresh_store=0 if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then case "${fetch_source}" in @@ -116,11 +115,8 @@ if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED: esac fi -if [ "${refresh_store}" -eq 1 ]; then - case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${fetch_source} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; - esac - [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." +if [ "${refresh_store}" -eq 1 ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ]; then + fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." fi if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then @@ -131,16 +127,6 @@ fi "$(dirname "${BASH_SOURCE[0]}")/vortex-login-container-registry" -if [ "${refresh_store}" -eq 1 ]; then - # The dump fetched from the store is removed, so the check below proves the - # upstream source wrote a new one. - rm -f "${db_file}" - # CircleCI re-sources the job variables from BASH_ENV in every bash - # script, which would replace the prefixed values. - env BASH_ENV='' "VORTEX_FETCH_DB_SOURCE=${VORTEX_REFRESH_DB_SOURCE}" "VORTEX_FETCH_DB_FORCE=1" "$(dirname "${BASH_SOURCE[0]}")/vortex-fetch-db" - [ -f "${db_file}" ] || fail "The database dump file was not fetched to ${db_file}. Remove the output directory and file name overrides of the database fetch source." -fi - # The value of VORTEX_DB_IMAGE passed to the provisioning. An empty value # makes the provisioning import the dump file. provision_db_image="${db_image}" diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index c274716c30..17adbc8f7e 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -326,6 +326,91 @@ EOF popd >/dev/null } +@test "fetch-db: Fetch a dump store source from the refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + for db_source in gdrive s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the lagoon source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from lagoon." + assert_output_not_contains "Fetched from ${db_source}." + assert_output_contains "Finished database fetch." + done + + popd >/dev/null +} + +@test "fetch-db: Fetch a dump store source from the store without a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + unset VORTEX_PUSH_DB_PROCEED + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetched from gdrive." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fetch an indexed dump store source from the store with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_SOURCE="gdrive" + export VORTEX_FETCH_DB3_PROCEED="1" + export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + # Only the primary database is pushed to a dump store. + assert_output_contains "Fetched from gdrive." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + +@test "fetch-db: Fail without an upstream source for a dump store source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_PROCEED="1" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" + + for refresh_source in "" gdrive s3 container_registry none; do + export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + + run .vortex/tooling/src/vortex-fetch-db + assert_failure + assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." + assert_output_not_contains "Fetched from" + assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" + done + + popd >/dev/null +} + @test "fetch-db: Fetch nothing with the none source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 338b9a7e8a..785087de4a 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -246,23 +246,20 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Fetches the dump file from the refresh source for a dump store source with a push requested" { +@test "refresh-db: Sanitizes the database for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + mkdir -p .data + echo "upstream" >.data/db.sql + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" - touch "${BUILD_DIR}/bash_env" - export BASH_ENV="${BUILD_DIR}/bash_env" stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\necho "Fetched from ${VORTEX_FETCH_DB_SOURCE} with force ${VORTEX_FETCH_DB_FORCE} and BASH_ENV [${BASH_ENV}]."\n' >.vortex/tooling/src/vortex-fetch-db - chmod +x .vortex/tooling/src/vortex-fetch-db export VORTEX_FETCH_DB_SOURCE="gdrive" - mkdir -p .data - echo "store" >.data/db.sql mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -275,7 +272,8 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_success - assert_output_contains "Fetched from lagoon with force 1 and BASH_ENV []." + # The fetch step already fetched the dump from the upstream source. + assert_output_not_contains "Fetched the dump file." assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." assert_output_contains "Sanitized the database." assert_output_contains "Exported the database to db.sql." @@ -288,22 +286,19 @@ load ../_helper.bash assert_string_contains "$(mock_get_call_args "${mock_docker}" 6)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" assert_equal "6" "$(mock_get_call_num "${mock_docker}")" - assert_file_contains .data/db.sql "upstream" - popd >/dev/null } -@test "refresh-db: Provisions from the dump store without the refresh source when a push is not requested" { +@test "refresh-db: Exports the dump store database unsanitized when a push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "store" >.data/db.sql - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED VORTEX_REFRESH_DB_SOURCE + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED export VORTEX_FETCH_DB_SOURCE="gdrive" stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' mock_command "sleep" >/dev/null @@ -311,39 +306,12 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_success - assert_output_not_contains "Fetched the dump file." assert_output_contains "Exported the database to db.sql." assert_output_contains "[ OK ] Finished database refresh." + # The dump from the store is already sanitized. + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "compose --progress quiet cp -L cli:/app/.data/db.sql ./.data/db.sql" assert_equal "5" "$(mock_get_call_num "${mock_docker}")" - assert_file_contains .data/db.sql "store" - - popd >/dev/null -} - -@test "refresh-db: Fails without an upstream source for a dump store source with a push requested" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" - export VORTEX_PUSH_DB_PROCEED=1 - - stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." - - mock_docker=$(mock_command "docker") - - for refresh_source in "" gdrive s3 container_registry none; do - export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" - - run .vortex/tooling/src/vortex-refresh-db - assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Logged in to the registry." - assert_output_not_contains "Fetched the dump file." - done - - assert_equal "0" "$(mock_get_call_num "${mock_docker}")" popd >/dev/null } @@ -354,18 +322,16 @@ load ../_helper.bash unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_FETCH_DB_SOURCE="gdrive" export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file." mock_docker=$(mock_command "docker") run .vortex/tooling/src/vortex-refresh-db assert_failure assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." - assert_output_not_contains "Fetched the dump file." + assert_output_not_contains "Logged in to the registry." assert_equal "0" "$(mock_get_call_num "${mock_docker}")" popd >/dev/null @@ -374,15 +340,15 @@ load ../_helper.bash @test "refresh-db: Fails when the sanitization fails for a dump store source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + mkdir -p .data + echo "upstream" >.data/db.sql + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="s3" export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="url" stub_sibling "vortex-login-container-registry" "Logged in to the registry." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - printf '#!/usr/bin/env bash\nmkdir -p .data\necho upstream >.data/db.sql\n' >.vortex/tooling/src/vortex-fetch-db - chmod +x .vortex/tooling/src/vortex-fetch-db mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -400,34 +366,6 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Fails when the refresh source does not fetch the dump file for a dump store source" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "store" >.data/db.sql - - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="s3" - export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="url" - - stub_sibling "vortex-login-container-registry" "Logged in to the registry." - stub_sibling "vortex-fetch-db" "Fetched the dump file elsewhere." - - mock_docker=$(mock_command "docker") - - run .vortex/tooling/src/vortex-refresh-db - assert_failure - assert_output_contains "Fetched the dump file elsewhere." - assert_output_contains "[FAIL] The database dump file was not fetched to ./.data/db.sql. Remove the output directory and file name overrides of the database fetch source." - - # The stale dump from the store is never provisioned or pushed. - assert_file_not_exists .data/db.sql - assert_equal "0" "$(mock_get_call_num "${mock_docker}")" - - popd >/dev/null -} - @test "refresh-db: Fails when the exported dump file cannot be copied from the cli container" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 From facf98ed2fd022aaf16038c15ccc55294b7bbf59 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:42:46 +1100 Subject: [PATCH 06/40] [#154] Narrowed the fetch-db upstream source failure assertion. --- .vortex/tooling/tests/unit/fetch-db.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index 17adbc8f7e..bd8d08e665 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -404,7 +404,8 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Fetched from" + assert_output_not_contains "Fetched from gdrive." + assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" done From 08e24e38e54ca66bae1a80cc1bbadfade2c815b5 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:44:14 +1100 Subject: [PATCH 07/40] [#154] Regenerated the variables reference. --- .vortex/docs/content/development/variables.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index cb724c71b3..5ff07170f8 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -399,7 +399,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | -| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `UNDEFINED` | `.env.local.example` | +| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -570,7 +570,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Upstream fetch source of the database dump file.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE.

Also required when the fetch source is a dump store ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED is set to `1`: the dump pushed to the store comes from an upstream source such as 'url', 'ftp', 'acquia' or 'lagoon'. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | From d10d3de7dab0556ab9336ec4b69bb4f91f25d84d Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 10:46:18 +1100 Subject: [PATCH 08/40] Updated snapshots. --- .../handler_process/db_fetch_source_gdrive/.env.local.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example index e62dbc8c32..0f1fe53eb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example @@ -6,5 +6,5 @@ +# Database dump file sourced from Google Drive. +# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key + -+# Google Cloud service account key: the JSON content or a path to the JSON key file. ++# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. +VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= From d0e8c9fcc9b3632b1d911c4bec7edb4e545110d6 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 11:19:17 +1100 Subject: [PATCH 09/40] Addressed code review: defaulted the Google Drive and S3 push file name to 'VORTEX_FETCH_DB_FILE'. --- .vortex/tooling/src/vortex-push-db-gdrive | 5 ++- .vortex/tooling/src/vortex-push-db-s3 | 4 +- .../tooling/tests/unit/push-db-gdrive.bats | 31 +++++++++++++++ .vortex/tooling/tests/unit/push-db-s3.bats | 38 +++++++++++++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive index 0f153bc016..c2a975752a 100755 --- a/.vortex/tooling/src/vortex-push-db-gdrive +++ b/.vortex/tooling/src/vortex-push-db-gdrive @@ -32,8 +32,9 @@ VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-. # Database dump file name. # -# The same name is used for the file in the Google Drive folder. -VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" +# Defaults to the name the database is fetched and refreshed with. The same +# name is used for the file in the Google Drive folder. +VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" # ------------------------------------------------------------------------------ diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 569555f344..fd6986b1ff 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -35,7 +35,9 @@ VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFI VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" # Database dump file name. -VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}" +# +# Defaults to the name the database is fetched and refreshed with. +VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" # Remote database dump file name. # diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats index 0d7e6b4cc6..35773af729 100644 --- a/.vortex/tooling/tests/unit/push-db-gdrive.bats +++ b/.vortex/tooling/tests/unit/push-db-gdrive.bats @@ -295,3 +295,34 @@ fixture_gdrive_key() { popd >/dev/null } + +@test "push-db-gdrive: Push the database dump file the database is fetched with" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db_fetched.sql + fixture_gdrive_key + + export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" + export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" + export VORTEX_FETCH_DB_FILE="db_fetched.sql" + unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 + mock_set_output "${mock_curl}" '{"files":[]}' 2 + mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-5\r\n' 3 + mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 + + run .vortex/tooling/src/vortex-push-db-gdrive + assert_success + assert_output_contains "Local file: ./.data/db_fetched.sql" + assert_output_contains "Remote file: db_fetched.sql" + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db_fetched.sql' and" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "--upload-file ./.data/db_fetched.sql" + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 6c7ccbe3d3..c467452983 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -471,3 +471,41 @@ load ../_helper.bash popd >/dev/null } + +@test "push-db-s3: Push the database dump file the database is fetched with" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db_fetched.sql + + declare -a STEPS=( + "[INFO] Started database dump push to S3." + + "Local file: ./.data/db_fetched.sql" + "Remote file: db_fetched.sql" + + "@curl * # 0 # 200" + + "[ OK ] Finished database dump push to S3." + ) + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_FETCH_DB_FILE="db_fetched.sql" + unset VORTEX_PUSH_DB_S3_DB_DIR VORTEX_PUSH_DB_S3_DB_FILE VORTEX_PUSH_DB_S3_REMOTE_FILE + + mocks="$(steps_run "setup")" + run .vortex/tooling/src/vortex-push-db-s3 + steps_run "assert" "${mocks[@]}" + + assert_success + + rm -rf .data + + popd >/dev/null +} From 6234c31f25489a1f1be23848f647e289fe9ea7ed Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 11:21:07 +1100 Subject: [PATCH 10/40] Addressed code review: regenerated the variables reference. --- .vortex/docs/content/development/variables.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 5ff07170f8..d047a26eae 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -556,14 +556,14 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | | `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | From 8c6abc175b4ce267fe336f01267a75f9ef42cc8c Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:34:51 +1100 Subject: [PATCH 11/40] Replaced the Google Drive push with an FTP push that keeps the current and the previous dump, for each database. --- .../db_fetch_source_gdrive/.env | 45 --- .../db_fetch_source_gdrive/.env.local.example | 10 - .../.github/workflows/build-test-deploy.yml | 33 -- .../migration_fetch_source_gdrive/.ahoy.yml | 51 --- .../migration_fetch_source_gdrive/.env | 30 -- .../.github/workflows/build-test-deploy.yml | 22 -- .../composer.json | 9 - .../docker-compose.yml | 52 --- .../scripts/provision-20-migration.sh | 187 ---------- .../phpunit/Drupal/DatabaseSettingsTest.php | 110 ------ .../Drupal/EnvironmentSettingsTest.php | 14 - .../tests/phpunit/Drupal/SettingsTestCase.php | 8 - .../migrations/ys_migrate_categories.yml | 29 -- .../custom/ys_migrate/ys_migrate.info.yml | 9 - .../web/sites/default/settings.migration.php | 21 -- .../web/sites/default/settings.php | 15 - .../tooling/playground/try-fetch-db-gdrive.sh | 57 --- .vortex/tooling/playground/try-push-db-ftp.sh | 71 ++++ .../tooling/playground/try-push-db-gdrive.sh | 65 ---- .vortex/tooling/src/vortex-fetch-db | 24 +- .vortex/tooling/src/vortex-fetch-db-gdrive | 135 ------- .vortex/tooling/src/vortex-push-db | 39 ++- .vortex/tooling/src/vortex-push-db-ftp | 143 ++++++++ .vortex/tooling/src/vortex-push-db-gdrive | 161 --------- .vortex/tooling/src/vortex-push-db-s3 | 218 ++++++++---- .vortex/tooling/src/vortex-refresh-db | 28 +- .vortex/tooling/tests/_helper.bash | 1 + .../tooling/tests/unit/fetch-db-gdrive.bats | 296 ---------------- .vortex/tooling/tests/unit/fetch-db.bats | 63 ++-- .vortex/tooling/tests/unit/push-db-ftp.bats | 204 +++++++++++ .../tooling/tests/unit/push-db-gdrive.bats | 328 ------------------ .vortex/tooling/tests/unit/push-db-s3.bats | 145 ++++++++ .vortex/tooling/tests/unit/push-db.bats | 86 ++++- .vortex/tooling/tests/unit/refresh-db.bats | 40 ++- 34 files changed, 916 insertions(+), 1833 deletions(-) delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml delete mode 100755 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php delete mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php delete mode 100755 .vortex/tooling/playground/try-fetch-db-gdrive.sh create mode 100755 .vortex/tooling/playground/try-push-db-ftp.sh delete mode 100755 .vortex/tooling/playground/try-push-db-gdrive.sh delete mode 100755 .vortex/tooling/src/vortex-fetch-db-gdrive create mode 100755 .vortex/tooling/src/vortex-push-db-ftp delete mode 100755 .vortex/tooling/src/vortex-push-db-gdrive delete mode 100644 .vortex/tooling/tests/unit/fetch-db-gdrive.bats create mode 100644 .vortex/tooling/tests/unit/push-db-ftp.bats delete mode 100644 .vortex/tooling/tests/unit/push-db-gdrive.bats diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env deleted file mode 100644 index eb9fa4209c..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env +++ /dev/null @@ -1,45 +0,0 @@ -@@ -180,13 +180,22 @@ - VORTEX_DB_FILE=db.sql - - # Database fetch source. --VORTEX_FETCH_DB_SOURCE=url -+VORTEX_FETCH_DB_SOURCE=gdrive - --# Database dump file sourced from a URL. -+# Database dump file sourced from Google Drive. -+ -+# Google Drive folder ID with the database dump file. - # --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= -+# The ID is the last segment of the folder URL. The folder must be in a shared -+# drive and shared with the service account. -+VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= - -+# Source of the database dump file the nightly CI job sanitizes and pushes to -+# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -+# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -+# Add them, and keep secrets in CI. -+# VORTEX_REFRESH_DB_SOURCE=url -+ - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +263,3 @@ - # with optional names in the format "email|name". - # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" - VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" -- --################################################################################ --# DEMO # --################################################################################ --# # --# Override project-specific values for demonstration purposes. # --# Used to showcase Vortex without asking users to perform additional steps. # --# # --# Remove this section after completing database fetch integration. # --# # --################################################################################ -- --# URL of the database used for demonstration with URL database fetch type. --VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example deleted file mode 100644 index 0f1fe53eb1..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.env.local.example +++ /dev/null @@ -1,10 +0,0 @@ -@@ -32,3 +32,9 @@ - - # Always override existing fetched DB dump. - VORTEX_FETCH_DB_FORCE=1 -+ -+# Database dump file sourced from Google Drive. -+# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key -+ -+# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. -+VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml deleted file mode 100644 index 88c93392b7..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_gdrive/.github/workflows/build-test-deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -@@ -191,6 +191,7 @@ - VORTEX_CI_DB_CACHE_FALLBACK: "yes" - # Which branch to use as a source of DB caches. - VORTEX_CI_DB_CACHE_BRANCH: "develop" -+ VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - - steps: - # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -241,6 +242,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" -+ # Refresh the database dump in the database fetch source. -+ echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +277,15 @@ - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi - ./vendor/bin/vortex-refresh-db -+ timeout-minutes: 30 -+ -+ # Push the freshly-exported sanitized database dump to the database fetch -+ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the -+ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). -+ - name: Push DB -+ run: | -+ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi -+ ./vendor/bin/vortex-push-db - timeout-minutes: 30 - - # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml deleted file mode 100644 index 11f7a073e6..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.ahoy.yml +++ /dev/null @@ -1,51 +0,0 @@ -@@ -32,6 +32,7 @@ - cmd: | - export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-${PWD##*/}} - export VORTEX_HOST_DB_PORT=$(docker compose port database 3306 2>/dev/null | cut -d : -f 2) -+ export VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_SOLR_PORT=$(docker compose port solr 8983 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_SELENIUM_VNC_PORT=$(docker compose port chrome 7900 2>/dev/null | cut -d : -f 2) - export VORTEX_HOST_HAS_SEQUELACE=$(uname -a | grep -i -q darwin && mdfind -name 'Sequel Ace' 2>/dev/null | grep -q "Ace" && echo 1 || true) -@@ -45,6 +46,14 @@ - open "mysql://${DATABASE_USERNAME:-drupal}:${DATABASE_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB_PORT}/drupal" -a "Sequel Ace" \ - || echo "Not a supported OS or Sequel Ace is not installed." - -+ db2: -+ usage: Open DB2 in Sequel Ace. -+ cmd: | -+ uname -a | grep -i -q darwin && test -d "${VORTEX_HOST_SEQUELACE_PATH:-/Applications/Sequel Ace.app}" && \ -+ VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) && \ -+ open "mysql://${DATABASE2_USERNAME:-drupal}:${DATABASE2_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB2_PORT}/drupal" -a "Sequel Ace" \ -+ || echo "Not a supported OS or Sequel Ace is not installed." -+ - # ---------------------------------------------------------------------------- - # Container commands. - # ---------------------------------------------------------------------------- -@@ -122,6 +131,13 @@ - case " $* " in *" --fresh "*) export VORTEX_FETCH_DB_FRESH=1 VORTEX_FETCH_DB_FORCE=1;; esac - ./vendor/bin/vortex-fetch-db - -+ fetch-db2: -+ usage: Fetch second database (migration). Run with "--fresh" to force. -+ aliases: [download-db2, db2-download, db2-fetch] -+ cmd: | -+ case " $* " in *" --fresh "*) export VORTEX_FETCH_DB2_FRESH=1 VORTEX_FETCH_DB2_FORCE=1;; esac -+ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db -+ - fetch-files: - usage: Fetch public files. - aliases: [download-files, files-download, files-fetch] -@@ -134,6 +150,13 @@ - docker compose rm --force --stop --volumes database - docker compose build database - ahoy up -+ -+ reload-db2: -+ usage: Reload the migration database container using local database image. -+ cmd: | -+ ahoy confirm "Running this command will replace your current database. Are you sure?" || exit 0 -+ docker compose rm --force --stop --volumes database2 -+ ahoy up -- --build database2 - - provision: - usage: Provision a site from the database dump or profile. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env deleted file mode 100644 index bce43c946e..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.env +++ /dev/null @@ -1,30 +0,0 @@ -@@ -195,6 +195,29 @@ - VORTEX_FETCH_DB_ENVIRONMENT=prod - - ################################################################################ -+# SECOND DATABASE (MIGRATION) # -+################################################################################ -+ -+# Second database for migrations. -+# See settings.migration.php for database credentials. -+ -+# Second database dump file name. -+VORTEX_FETCH_DB2_FILE=db2.sql -+ -+# Second database fetch source. -+VORTEX_FETCH_DB2_SOURCE=gdrive -+ -+# Second database dump file sourced from Google Drive. -+ -+# Google Drive folder ID with the second database dump file. -+VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= -+ -+# Environment to fetch the second database from. -+# -+# Applies to hosting environments. -+VORTEX_FETCH_DB2_ENVIRONMENT=prod -+ -+################################################################################ - # FILES SOURCE # - ################################################################################ - diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml deleted file mode 100644 index 4292b4f549..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/.github/workflows/build-test-deploy.yml +++ /dev/null @@ -1,22 +0,0 @@ -@@ -270,6 +270,9 @@ - echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" - timeout-minutes: 30 - -+ - name: Fetch migration DB -+ run: VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db -+ - - name: Export DB - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ - docker compose exec cli mkdir -p .data - docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql - rm -f .data/db.sql -+ fi -+ if [ -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ]; then -+ docker compose exec -T cli mkdir -p .data -+ docker compose --progress quiet cp -L ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" cli:"/app/.data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" -+ rm -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" - fi - docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision - timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json deleted file mode 100644 index b90a23c71c..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/composer.json +++ /dev/null @@ -1,9 +0,0 @@ -@@ -26,6 +26,8 @@ - "drupal/linkit": "__VERSION__", - "drupal/menu_trail_by_path": "__VERSION__", - "drupal/metatag": "__VERSION__", -+ "drupal/migrate_plus": "__VERSION__", -+ "drupal/migrate_tools": "__VERSION__", - "drupal/navigation_extra_tools": "__VERSION__", - "drupal/pathauto": "__VERSION__", - "drupal/redirect": "__VERSION__", diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml deleted file mode 100644 index 70ac2fcbc8..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -@@ -57,6 +57,16 @@ - DRUPAL_SHIELD_PASS: ${DRUPAL_SHIELD_PASS:-} - # Redis integration flag. - DRUPAL_REDIS_ENABLED: ${DRUPAL_REDIS_ENABLED:-} -+ # Local migration database host (not used in production). -+ DATABASE2_HOST: database2 -+ # Local migration database name (not used in production). -+ DATABASE2_NAME: drupal -+ # Local migration database user (not used in production). -+ DATABASE2_USERNAME: drupal -+ # Local migration database password (not used in production). -+ DATABASE2_PASSWORD: drupal -+ # Local migration database port (not used in production). -+ DATABASE2_PORT: 3306 - - # The default user under which the containers should run. - x-user: &default-user -@@ -155,6 +165,20 @@ - ports: - - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database 3306`. - -+ database2: -+ image: "${VORTEX_DB2_IMAGE:-uselagoon/mysql-8.4:__VERSION__}" # Use custom database image (if defined) or fallback to standard database image. -+ environment: -+ <<: *default-environment -+ MYSQL_DATABASE: drupal -+ MYSQL_USER: drupal -+ MYSQL_PASSWORD: drupal -+ <<: *default-user -+ # dclint disable-line no-duplicate-exported-ports # container-only port, published on a random host port -+ ports: -+ - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database2 3306`. -+ labels: -+ lagoon.type: mariadb # See https://docs.lagoon.sh/using-lagoon-advanced/service-types/ -+ - redis: - image: uselagoon/valkey-8:__VERSION__ - -@@ -198,11 +222,13 @@ - depends_on: - - clamav - - database -+ - database2 - - redis - - solr - command: - - clamav:3310 - - database:3306 -+ - database2:3306 - - redis:6379 - - solr:8983 - - curl -s "http://solr:8983/solr/drupal/select?q=*:*&rows=0&wt=json" | grep response diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh deleted file mode 100755 index 3000594760..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/scripts/provision-20-migration.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -## -# Import migration database and run migrations. -# -# This script is called during site provisioning via provision.sh. -# Customize the migration names at the bottom of this file. -# -# shellcheck disable=SC2086 - -set -eu -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -# Skip all migrations. -DRUPAL_MIGRATION_SKIP="${DRUPAL_MIGRATION_SKIP:-0}" - -# Skip rollback of migrations before import. -DRUPAL_MIGRATION_ROLLBACK_SKIP="${DRUPAL_MIGRATION_ROLLBACK_SKIP:-1}" - -# Limit the number of entities to import. Set to 'all' to import all. -DRUPAL_MIGRATION_IMPORT_LIMIT="${DRUPAL_MIGRATION_IMPORT_LIMIT:-50}" - -# Update already imported entities during migration. -DRUPAL_MIGRATION_UPDATE="${DRUPAL_MIGRATION_UPDATE:-0}" - -# Feedback frequency for migration progress. -DRUPAL_MIGRATION_FEEDBACK="${DRUPAL_MIGRATION_FEEDBACK:-50}" - -# Import migration source database. Set to 1 to import, 0 to skip. -DRUPAL_MIGRATION_SOURCE_DB_IMPORT="${DRUPAL_MIGRATION_SOURCE_DB_IMPORT:-${VORTEX_PROVISION_OVERRIDE_DB:-0}}" - -# Table name to probe in the source database to verify it is not corrupted. -DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE="${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE:-categories}" - -# Disable the search server while migrations run. Set to 0 to keep indexing. -DRUPAL_MIGRATION_SEARCH_DISABLE="${DRUPAL_MIGRATION_SEARCH_DISABLE:-1}" - -# Directory with database dump file. -VORTEX_DB_DIR="${VORTEX_DB_DIR:-./.data}" - -# Migration database dump file name. -VORTEX_FETCH_DB2_FILE="${VORTEX_FETCH_DB2_FILE:-db2.sql}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { printf " ==> %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { printf " > %s\n" "${1}"; } -pass() { printf " + %s\n" "${1}"; } -fail() { printf " ! %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -drush() { ./vendor/bin/drush -y "$@"; } - -# ------------------------------------------------------------------------------ - -info "Started migration operations." - -environment="$(drush php:eval "print \Drupal\Core\Site\Settings::get('environment');")" -note "Environment: ${environment}" - -# Skip migrations in production. -if [ "${environment}" = "prod" ]; then - DRUPAL_MIGRATION_SKIP=1 -fi - -note "Migration skip: ${DRUPAL_MIGRATION_SKIP}" -note "Migration limit: ${DRUPAL_MIGRATION_IMPORT_LIMIT}" -note "Migration skip rollback: ${DRUPAL_MIGRATION_ROLLBACK_SKIP}" -note "Migration update: ${DRUPAL_MIGRATION_UPDATE}" -note "Migration feedback: ${DRUPAL_MIGRATION_FEEDBACK}" -note "Source DB import: ${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" -note "Search disable: ${DRUPAL_MIGRATION_SEARCH_DISABLE}" -echo - -if [ "${DRUPAL_MIGRATION_SKIP}" = "1" ]; then - info "Skipped migrations. DRUPAL_MIGRATION_SKIP is set to 1." - exit 0 -fi - -# Helper function to run a single migration with configured options. -run_migration() { - local migration_name="${1}" - shift - - drush migrate:reset-status "${migration_name}" || fail "Failed to reset migration status for ${migration_name}." - - task "Running migration: ${migration_name}." - local opts=() - - opts+=("--feedback=${DRUPAL_MIGRATION_FEEDBACK}") - - if printf '%s\n' "${DRUPAL_MIGRATION_IMPORT_LIMIT}" | grep -q '^[0-9][0-9]*$' && [ "${DRUPAL_MIGRATION_IMPORT_LIMIT}" -gt 0 ]; then - opts+=("--limit=${DRUPAL_MIGRATION_IMPORT_LIMIT}") - fi - - if [ "${DRUPAL_MIGRATION_UPDATE}" = "1" ]; then - opts+=("--update") - fi - - # Add any additional arguments passed to the function. - opts+=("$@") - - drush migrate:import "${opts[@]}" "${migration_name}" || { - drush migrate:messages "${migration_name}" - fail "Failed to run migration ${migration_name}." - } - - pass "Migrated: ${migration_name}." -} - -# Detect if existing migration source database is corrupted. -if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" != "1" ]; then - note "Source database import is set to be skipped. Checking existing database." - task "Probing for '${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}' table in the source database." - if drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then - pass "Source database is intact." - else - note "Migration source database is corrupted or empty. Re-importing." - DRUPAL_MIGRATION_SOURCE_DB_IMPORT=1 - pass "Marked the source database for re-import." - fi -fi - -# Import the migration source database from the dump file. -if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" = "1" ]; then - task "Importing migration source database." - - [ ! -f "${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" ] && fail "Migration source database file not found. Please run 'ahoy fetch-db2'." - - drush sql:drop --database=migrate - # shellcheck disable=SC2091 - $(drush sql:connect --database=migrate) <"${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" - - pass "Imported migration source database." -else - note "Using existing migration source database." -fi - -task "Verifying migration source database." -if ! drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then - drush sql:query --database=migrate "SHOW TABLES;" - fail "Migration source database is corrupted." -fi -pass "Verified migration source database." - -# Enable custom migration modules. -task "Enabling migration modules." -drush pm:install ys_migrate -pass "Enabled migration modules." - -info "Starting migrations." - -if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then - task "Disabling Search API Solr server." - drush search-api:server-disable solr || true - pass "Disabled Search API Solr server." -fi - -if [ "${DRUPAL_MIGRATION_ROLLBACK_SKIP}" = "1" ]; then - note "Skipped rollback of all migrations." -else - task "Rolling back all migrations." - drush migrate:rollback --all || true - pass "Rolled back all migrations." -fi -echo - -# ------------------------------------------------------------------------------ -# Add your migrations below. -# ------------------------------------------------------------------------------ - -run_migration ys_migrate_categories - -echo -note "Finished migrations." - -if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then - task "Enabling Search API Solr server." - drush search-api:server-enable solr || true - drush search-api:enable-all || true - pass "Enabled Search API Solr server." -fi - -drush migrate:status - -info "Finished migration operations." diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php deleted file mode 100644 index 956cd535d4..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/DatabaseSettingsTest.php +++ /dev/null @@ -1,110 +0,0 @@ -@@ -47,6 +47,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -74,6 +85,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -101,6 +123,17 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -@@ -128,9 +161,55 @@ - 'prefix' => '', - ], - ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], - ], - ]; - -+ yield [ -+ [ -+ 'DATABASE2_NAME' => 'migrate_db_name', -+ 'DATABASE2_USERNAME' => 'migrate_db_user', -+ 'DATABASE2_PASSWORD' => 'migrate_db_pass', -+ 'DATABASE2_HOST' => 'migrate_db_host', -+ 'DATABASE2_PORT' => '3307', -+ ], -+ [ -+ 'default' => [ -+ 'default' => [ -+ 'database' => 'drupal', -+ 'username' => 'drupal', -+ 'password' => 'drupal', -+ 'host' => 'localhost', -+ 'port' => '3306', -+ 'charset' => 'utf8mb4', -+ 'collation' => 'utf8mb4_general_ci', -+ 'driver' => 'mysql', -+ 'prefix' => '', -+ ], -+ ], -+ 'migrate' => [ -+ 'default' => [ -+ 'database' => 'migrate_db_name', -+ 'username' => 'migrate_db_user', -+ 'password' => 'migrate_db_pass', -+ 'host' => 'migrate_db_host', -+ 'port' => '3307', -+ 'prefix' => '', -+ 'driver' => 'mysql', -+ ], -+ ], -+ ], -+ ]; - } - - } diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php deleted file mode 100644 index 8adb9246e8..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/EnvironmentSettingsTest.php +++ /dev/null @@ -1,14 +0,0 @@ -@@ -163,6 +163,13 @@ - $databases['default']['default']['collation'] = 'utf8_general_ci'; - $databases['default']['default']['driver'] = 'mysql'; - $databases['default']['default']['prefix'] = ''; -+ $databases['migrate']['default']['database'] = 'drupal'; -+ $databases['migrate']['default']['username'] = 'drupal'; -+ $databases['migrate']['default']['password'] = 'drupal'; -+ $databases['migrate']['default']['host'] = 'localhost'; -+ $databases['migrate']['default']['port'] = ''; -+ $databases['migrate']['default']['prefix'] = ''; -+ $databases['migrate']['default']['driver'] = 'mysql'; - $this->assertEquals($databases, $this->databases); - - // Verify key config overrides. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php deleted file mode 100644 index 350cc85c04..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/tests/phpunit/Drupal/SettingsTestCase.php +++ /dev/null @@ -1,8 +0,0 @@ -@@ -62,6 +62,7 @@ - const ALLOWED_ENV_VARS = [ - // Service variables. - 'DATABASE_', -+ 'DATABASE2_', - 'REDIS_', - 'CLAMAV_', - 'COMPOSE_', diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml deleted file mode 100644 index 0494698e5d..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml +++ /dev/null @@ -1,29 +0,0 @@ -id: ys_migrate_categories -label: 'Migrate categories to tags' -migration_tags: - - ys_migrate - -source: - plugin: table - key: migrate - table_name: categories - id_fields: - id: - type: integer - fields: - id: id - name: name - description: description - -process: - name: name - description/value: description - description/format: - plugin: default_value - default_value: plain_text - vid: - plugin: default_value - default_value: tags - -destination: - plugin: 'entity:taxonomy_term' diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml deleted file mode 100644 index 96c67179ac..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/modules/custom/ys_migrate/ys_migrate.info.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: star wars Migrate -type: module -description: Migrations for star wars site. -core_version_requirement: ^11 -package: star_wars -dependencies: - - drupal:migrate - - migrate_plus:migrate_plus - - migrate_tools:migrate_tools diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php deleted file mode 100644 index 660869b39f..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.migration.php +++ /dev/null @@ -1,21 +0,0 @@ - (getenv('DATABASE2_NAME') ?: getenv('DATABASE2_DATABASE')) ?: 'drupal', - 'username' => getenv('DATABASE2_USERNAME') ?: 'drupal', - 'password' => getenv('DATABASE2_PASSWORD') ?: 'drupal', - 'host' => getenv('DATABASE2_HOST') ?: 'localhost', - 'port' => getenv('DATABASE2_PORT') ?: '', - 'prefix' => '', - 'driver' => 'mysql', -]; diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php deleted file mode 100644 index c04331d75e..0000000000 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_gdrive/web/sites/default/settings.php +++ /dev/null @@ -1,15 +0,0 @@ -@@ -167,6 +167,14 @@ - } - - //////////////////////////////////////////////////////////////////////////////// -+/// MIGRATION SETTINGS /// -+//////////////////////////////////////////////////////////////////////////////// -+// @todo Migration. Remove when migration configuration is no longer required. -+if (file_exists($app_root . '/' . $site_path . '/settings.migration.php')) { -+ require $app_root . '/' . $site_path . '/settings.migration.php'; -+} -+ -+//////////////////////////////////////////////////////////////////////////////// - /// LOCAL OVERRIDE /// - //////////////////////////////////////////////////////////////////////////////// - // @see https://www.vortextemplate.com/docs/development/settings#local-overrides diff --git a/.vortex/tooling/playground/try-fetch-db-gdrive.sh b/.vortex/tooling/playground/try-fetch-db-gdrive.sh deleted file mode 100755 index 43fd135f42..0000000000 --- a/.vortex/tooling/playground/try-fetch-db-gdrive.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -## -# Manual test script for Google Drive database fetch. -# -# Fetches a database dump file from a Google Drive folder using -# fetch-db-gdrive. -# -# Usage: -# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" -# export GDRIVE_FOLDER_ID="your-folder-id" -# ./try-fetch-db-gdrive.sh - -set -eu - -# Source secrets if available. -if [ -f "$HOME/.profile.secrets" ]; then - # shellcheck disable=SC1091 - . "$HOME/.profile.secrets" -fi - -# Get the directory of this script and navigate to project root. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" - -# Google Drive credentials and settings. -GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" -GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" - -# Validate required variables. -if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then - echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" - echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" - exit 1 -fi - -if [ -z "${GDRIVE_FOLDER_ID}" ]; then - echo "Error: GDRIVE_FOLDER_ID environment variable is required" - echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" - exit 1 -fi - -echo "Testing Google Drive database fetch..." -echo "" -echo "Folder ID: ${GDRIVE_FOLDER_ID}" -echo "DB file: ${VORTEX_DB_FILE:-db.sql}" -echo "Local dir: ${VORTEX_DB_DIR:-./.data}" -echo "" - -cd "${PROJECT_ROOT}" || exit 1 - -export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" -export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" - -./vendor/drevops/vortex-tooling/src/vortex-fetch-db-gdrive - -echo "" -echo "Fetch complete!" diff --git a/.vortex/tooling/playground/try-push-db-ftp.sh b/.vortex/tooling/playground/try-push-db-ftp.sh new file mode 100755 index 0000000000..dda38af769 --- /dev/null +++ b/.vortex/tooling/playground/try-push-db-ftp.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +## +# Manual test script for FTP database push. +# +# Uploads a database dump file to an FTP server using push-db-ftp. Run it +# twice to check that the second run keeps the first dump as +# 'db-previous.sql' next to the new 'db.sql'. +# +# Usage: +# export FTP_HOST="ftp.example.com" +# export FTP_PORT="21" +# export FTP_USER="your-user" +# export FTP_PASS="your-password" +# export FTP_FILE="path/to/db.sql" +# ./try-push-db-ftp.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# FTP credentials and settings. +FTP_HOST="${FTP_HOST:-}" +FTP_PORT="${FTP_PORT:-21}" +FTP_USER="${FTP_USER:-}" +FTP_PASS="${FTP_PASS:-}" +FTP_FILE="${FTP_FILE:-db.sql}" + +# Validate required variables. +for name in FTP_HOST FTP_USER FTP_PASS; do + if [ -z "${!name}" ]; then + echo "Error: ${name} environment variable is required" + echo "Usage: export ${name}=\"...\" && $0" + exit 1 + fi +done + +cd "${PROJECT_ROOT}" || exit 1 + +# Create a test dump file when none exists. +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" +if [ ! -f "${db_dir}/${db_file}" ]; then + mkdir -p "${db_dir}" + echo "-- Test dump created by try-push-db-ftp.sh on $(date -u)" >"${db_dir}/${db_file}" +fi + +echo "Testing FTP database push..." +echo "" +echo "FTP host: ${FTP_HOST}:${FTP_PORT}" +echo "FTP file: ${FTP_FILE}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +export VORTEX_PUSH_DB_FTP_HOST="${FTP_HOST}" +export VORTEX_PUSH_DB_FTP_PORT="${FTP_PORT}" +export VORTEX_PUSH_DB_FTP_USER="${FTP_USER}" +export VORTEX_PUSH_DB_FTP_PASS="${FTP_PASS}" +export VORTEX_PUSH_DB_FTP_FILE="${FTP_FILE}" + +./vendor/drevops/vortex-tooling/src/vortex-push-db-ftp + +echo "" +echo "Push complete!" diff --git a/.vortex/tooling/playground/try-push-db-gdrive.sh b/.vortex/tooling/playground/try-push-db-gdrive.sh deleted file mode 100755 index 9cddbab46f..0000000000 --- a/.vortex/tooling/playground/try-push-db-gdrive.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -## -# Manual test script for Google Drive database push. -# -# Uploads a database dump file to a Google Drive folder in a shared drive -# using push-db-gdrive. Run it twice to check that the second run replaces -# the file instead of creating a duplicate. -# -# Usage: -# export GDRIVE_SERVICE_ACCOUNT_KEY="/path/to/service-account.json" -# export GDRIVE_FOLDER_ID="your-folder-id" -# ./try-push-db-gdrive.sh - -set -eu - -# Source secrets if available. -if [ -f "$HOME/.profile.secrets" ]; then - # shellcheck disable=SC1091 - . "$HOME/.profile.secrets" -fi - -# Get the directory of this script and navigate to project root. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" - -# Google Drive credentials and settings. -GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY:-}" -GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID:-}" - -# Validate required variables. -if [ -z "${GDRIVE_SERVICE_ACCOUNT_KEY}" ]; then - echo "Error: GDRIVE_SERVICE_ACCOUNT_KEY environment variable is required" - echo "Usage: export GDRIVE_SERVICE_ACCOUNT_KEY=\"/path/to/service-account.json\" && $0" - exit 1 -fi - -if [ -z "${GDRIVE_FOLDER_ID}" ]; then - echo "Error: GDRIVE_FOLDER_ID environment variable is required" - echo "Usage: export GDRIVE_FOLDER_ID=\"your-folder-id\" && $0" - exit 1 -fi - -cd "${PROJECT_ROOT}" || exit 1 - -# Create a test dump file when none exists. -db_dir="${VORTEX_DB_DIR:-./.data}" -db_file="${VORTEX_DB_FILE:-db.sql}" -if [ ! -f "${db_dir}/${db_file}" ]; then - mkdir -p "${db_dir}" - echo "-- Test dump created by try-push-db-gdrive.sh on $(date -u)" >"${db_dir}/${db_file}" -fi - -echo "Testing Google Drive database push..." -echo "" -echo "Folder ID: ${GDRIVE_FOLDER_ID}" -echo "Local file: ${db_dir}/${db_file}" -echo "" - -export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${GDRIVE_SERVICE_ACCOUNT_KEY}" -export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${GDRIVE_FOLDER_ID}" - -./vendor/drevops/vortex-tooling/src/vortex-push-db-gdrive - -echo "" -echo "Push complete!" diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index f1abf35058..b3f93c2f6e 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -55,17 +55,20 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 -# A dump store ('gdrive' or 's3') receives the dump that the nightly refresh -# pushes, so with a push requested the dump comes from its upstream source. -# The store may hold no dump yet on the first push. -if [ -z "${_db_index}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then +# An 'ftp' or 's3' source with an upstream source in VORTEX_REFRESH_DB_SOURCE +# is a dump store: it receives the dump that the nightly refresh pushes, so +# with a push requested the dump comes from the upstream source. The store +# may hold no dump yet on the first push. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +refresh_source="${!_v:-}" +if [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${refresh_source}" ]; then case "${VORTEX_FETCH_DB_SOURCE}" in - gdrive | s3) - case "${VORTEX_REFRESH_DB_SOURCE:-}" in - "" | gdrive | s3 | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE:-}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." ;; + ftp | s3) + case "${refresh_source}" in + "${VORTEX_FETCH_DB_SOURCE}" | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." ;; esac - note "Fetching from the ${VORTEX_REFRESH_DB_SOURCE} source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." - VORTEX_FETCH_DB_SOURCE="${VORTEX_REFRESH_DB_SOURCE}" + note "Fetching from the ${refresh_source} source in VORTEX_REFRESH_DB${_db_index}_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + VORTEX_FETCH_DB_SOURCE="${refresh_source}" ;; esac fi @@ -79,9 +82,8 @@ case "${VORTEX_FETCH_DB_SOURCE}" in lagoon) fetch_script="vortex-fetch-db-lagoon" ;; container_registry) fetch_script="vortex-fetch-db-container-registry" ;; s3) fetch_script="vortex-fetch-db-s3" ;; - gdrive) fetch_script="vortex-fetch-db-gdrive" ;; none) fetch_script="" ;; - *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none." ;; + *) fail "Unsupported database source \"${VORTEX_FETCH_DB_SOURCE}\" in VORTEX_FETCH_DB${_db_index}_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none." ;; esac # Skip file existence check for container_registry source as the database is diff --git a/.vortex/tooling/src/vortex-fetch-db-gdrive b/.vortex/tooling/src/vortex-fetch-db-gdrive deleted file mode 100755 index 546b40dc53..0000000000 --- a/.vortex/tooling/src/vortex-fetch-db-gdrive +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -## -# Fetch DB dump from Google Drive. -# -# Authenticates with a Google Cloud service account key and downloads the file -# with the database dump file name from a Google Drive folder. The folder may -# be in a shared drive. -# -# IMPORTANT! This script runs outside the container on the host system. -# -# shellcheck disable=SC1090,SC1091 - -t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t - -set -eu - -# Database index suffix. When set (e.g., "2"), all DB-related variable lookups -# use the indexed variant. -_db_index="${VORTEX_DB_INDEX:-}" - -# Google Cloud service account key: the JSON content or a path to the JSON -# key file. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY" -VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${!_v:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" - -# Google Drive folder ID with the database dump file. -# -# The ID is the last segment of the folder URL. The folder must be shared with -# the service account. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID" -VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="${!_v:-}" - -# Directory with database dump file. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_DIR" -_vs="VORTEX_FETCH_DB${_db_index}_DIR" -_vss="VORTEX_DB${_db_index}_DIR" -VORTEX_FETCH_DB_GDRIVE_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" - -# Database dump file name. -# -# The same name is used for the file in the Google Drive folder. -_v="VORTEX_FETCH_DB${_db_index}_GDRIVE_DB_FILE" -_vs="VORTEX_FETCH_DB${_db_index}_FILE" -_vss="VORTEX_DB${_db_index}_FILE" -VORTEX_FETCH_DB_GDRIVE_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } -pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } -fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -extract_json_last_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -extract_json_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -# Print a Google OAuth access token for a service account key and a scope. -# -# The key is passed to PHP in the environment, so it never appears in the -# process list. -gdrive_access_token() { - local key="${1}" - local scope="${2}" - - [ -f "${key}" ] && key="$(cat "${key}")" - - local assertion - # shellcheck disable=SC2016 - assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 - - local token_json - token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 - - echo "${token_json}" | extract_json_value "access_token" -} - -# Print the ID of the most recently modified file with a name in a folder. -# -# Prints nothing when the folder has no such file, and fails when the folder -# cannot be listed. -gdrive_find_file() { - local token="${1}" - local folder_id="${2}" - local name="${3}" - - local files_json - files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 - - local files - files=$(echo "${files_json}" | extract_json_value "files") || return 1 - - echo "${files}" | extract_json_last_value "id" || true -} - -for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done - -[ -z "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." -[ -z "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_FETCH_DB${_db_index}_GDRIVE_FOLDER_ID." - -info "Started database dump fetch from Google Drive." - -note "Remote file: ${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" -note "Local path: ${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" -note "Folder ID: ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" - -task "Authenticating with the service account." -token=$(gdrive_access_token "${VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive.readonly") || fail "Unable to authenticate with the service account. Check that VORTEX_FETCH_DB${_db_index}_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." -pass "Authenticated with the service account." - -task "Finding the database dump file in the folder." -file_id=$(gdrive_find_file "${token}" "${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_FETCH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." -[ -z "${file_id}" ] && fail "Unable to find the ${VORTEX_FETCH_DB_GDRIVE_DB_FILE} file in the ${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID} folder." -pass "Found the database dump file with ID ${file_id}." - -mkdir -p "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}" - -task "Downloading the database dump file." -curl -sS -f -L -H "Authorization: Bearer ${token}" -o "${VORTEX_FETCH_DB_GDRIVE_DB_DIR}/${VORTEX_FETCH_DB_GDRIVE_DB_FILE}" "https://www.googleapis.com/drive/v3/files/${file_id}?alt=media&supportsAllDrives=true" || fail "Unable to download the database dump file." -pass "Downloaded the database dump file." - -# Enable xtrace only after the authenticated requests so the access token is -# never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -pass "Finished database dump fetch from Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db index 4ba20e8c4d..9a36c9629f 100755 --- a/.vortex/tooling/src/vortex-push-db +++ b/.vortex/tooling/src/vortex-push-db @@ -5,8 +5,13 @@ # This is a router script to call relevant scripts based on type. # # The database dump file is pushed to the storage the database is fetched -# from, set in VORTEX_FETCH_DB_SOURCE. Only the 'gdrive' and 's3' sources -# accept a push; the push is skipped for other sources. +# from, set in VORTEX_FETCH_DB_SOURCE. Only an 'ftp' or 's3' source with an +# upstream source in VORTEX_REFRESH_DB_SOURCE accepts a push (a dump store); +# the push is skipped for other sources. +# +# With VORTEX_DB_INDEX set (e.g., "2" for the migration database), the +# indexed database pushes its own dump file with the indexed variables, so +# several databases share 1 storage. # # IMPORTANT! This script runs outside the container on the host system. # @@ -17,6 +22,10 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu [ "${VORTEX_DEBUG-}" = "1" ] && set -x +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant. +_db_index="${VORTEX_DB_INDEX:-}" + # Proceed with push. # # Set to 1 in the nightly CI job only, so the storage is refreshed once a day @@ -24,7 +33,12 @@ set -eu VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" # Database fetch source, which is also the push destination. -VORTEX_FETCH_DB_SOURCE="${VORTEX_FETCH_DB_SOURCE:-url}" +_v="VORTEX_FETCH_DB${_db_index}_SOURCE" +VORTEX_FETCH_DB_SOURCE="${!_v:-url}" + +# Upstream source of the pushed database dump file. +_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" +VORTEX_REFRESH_DB_SOURCE="${!_v:-}" # ------------------------------------------------------------------------------ @@ -36,19 +50,24 @@ pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START) fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } # @formatter:on -info "Started database push." +info "Started database${_db_index:+ ${_db_index}} push." [ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 case "${VORTEX_FETCH_DB_SOURCE}" in - gdrive) push_script="vortex-push-db-gdrive" ;; + ftp) push_script="vortex-push-db-ftp" ;; s3) push_script="vortex-push-db-s3" ;; - *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." && exit 0 ;; + *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB${_db_index}_SOURCE does not accept a push. Use one of: ftp, s3." && exit 0 ;; esac -# The pushed dump leaves the hosting and CI, so it must be sanitized. -[ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." +# Without an upstream source, the fetch source is where the dump comes from, +# and a push would replace that dump with its own export. +[ -z "${VORTEX_REFRESH_DB_SOURCE}" ] && pass "Skipped database push as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the ${VORTEX_FETCH_DB_SOURCE} source is not a dump store." && exit 0 + +# The pushed dump leaves the hosting and CI, so it must be sanitized. Only the +# primary database is sanitized: an indexed database is pushed as fetched. +[ -z "${_db_index}" ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." -"$(dirname "${BASH_SOURCE[0]}")/${push_script}" +env VORTEX_DB_INDEX="${_db_index}" "$(dirname "${BASH_SOURCE[0]}")/${push_script}" -pass "Finished database push." +pass "Finished database${_db_index:+ ${_db_index}} push." diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp new file mode 100755 index 0000000000..2979a1e1ab --- /dev/null +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +## +# Push DB dump to FTP. +# +# Keeps 2 copies of the database dump file on the FTP server: the current +# dump under the remote file name and the dump it replaced with a '-previous' +# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the +# current dump under the remote file name. +# +# The dump is uploaded under a temporary name and renamed into place only +# after the upload succeeds, so a failed upload keeps the current dump. +# +# IMPORTANT! This script runs outside the container on the host system. +# +# shellcheck disable=SC1090,SC1091 + +t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t + +set -eu + +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + +# The FTP values fall back to the values the database is fetched with, so a +# server used as the fetch source is configured once. + +# The FTP user. +_v="VORTEX_PUSH_DB${_db_index}_FTP_USER" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_USER" +VORTEX_PUSH_DB_FTP_USER="${!_v:-${!_vs:-}}" + +# The FTP password. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PASS" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PASS" +VORTEX_PUSH_DB_FTP_PASS="${!_v:-${!_vs:-}}" + +# The FTP host. +_v="VORTEX_PUSH_DB${_db_index}_FTP_HOST" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_HOST" +VORTEX_PUSH_DB_FTP_HOST="${!_v:-${!_vs:-}}" + +# The FTP port. +_v="VORTEX_PUSH_DB${_db_index}_FTP_PORT" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_PORT" +VORTEX_PUSH_DB_FTP_PORT="${!_v:-${!_vs:-21}}" + +# The remote file name, including any directories. +_v="VORTEX_PUSH_DB${_db_index}_FTP_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_FILE" +VORTEX_PUSH_DB_FTP_FILE="${!_v:-${!_vs:-}}" + +# Directory with database dump file. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_FTP_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" + +# Database dump file name. +# +# Defaults to the name the database is fetched and refreshed with. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_FTP_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" + +# Prefix for the names of the pushed database dump files. +# +# Use a different prefix in each place that pushes to the same storage, such +# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the +# prefixed name only when its remote file name includes the prefix. +VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } +pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } +fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +# Print a value quoted for a curl config file. +curl_config_quote() { + local value="${1//\\/\\\\}" + printf '"%s"' "${value//\"/\\\"}" +} + +# shellcheck disable=SC2043 +for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done + +[ -z "${VORTEX_PUSH_DB_FTP_USER}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_USER or VORTEX_FETCH_DB${_db_index}_FTP_USER." +[ -z "${VORTEX_PUSH_DB_FTP_PASS}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_PASS or VORTEX_FETCH_DB${_db_index}_FTP_PASS." +[ -z "${VORTEX_PUSH_DB_FTP_HOST}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_HOST or VORTEX_FETCH_DB${_db_index}_FTP_HOST." +[ -z "${VORTEX_PUSH_DB_FTP_FILE}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_FTP_FILE or VORTEX_FETCH_DB${_db_index}_FTP_FILE." + +local_file="${VORTEX_PUSH_DB_FTP_DB_DIR}/${VORTEX_PUSH_DB_FTP_DB_FILE}" +[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." + +remote_dir="" +remote_name="${VORTEX_PUSH_DB_FTP_FILE#/}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +current_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +if [[ ${remote_name} == *.* ]]; then + previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" +else + previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" +fi +upload_name="${current_name}.uploading" + +info "Started database${_db_index:+ ${_db_index}} dump push to FTP." + +note "Local file: ${local_file}" +note "Remote file: ${remote_dir}${current_name}" +note "Previous file: ${remote_dir}${previous_name}" +note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" + +# The credentials go to curl on stdin, so they never appear in the process +# list. The commands after the upload run in the remote directory: the +# previous dump is replaced by the current one, and the upload takes the +# current name. Only the final rename must succeed, since the first push has +# no current or previous dump. +task "Uploading the database dump file and keeping the dump it replaces." +curl_config="user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" +echo "${curl_config}" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ + --quote "-*DELE ${previous_name}" \ + --quote "-*RNFR ${current_name}" \ + --quote "-*RNTO ${previous_name}" \ + --quote "-RNFR ${upload_name}" \ + --quote "-RNTO ${current_name}" \ + "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" || fail "Unable to push the database dump file to FTP." +pass "Uploaded the database dump file and kept the dump it replaced." + +# Enable xtrace only after the request so the credentials are never written +# to the trace output. +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +pass "Finished database${_db_index:+ ${_db_index}} dump push to FTP." diff --git a/.vortex/tooling/src/vortex-push-db-gdrive b/.vortex/tooling/src/vortex-push-db-gdrive deleted file mode 100755 index c2a975752a..0000000000 --- a/.vortex/tooling/src/vortex-push-db-gdrive +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash -## -# Push DB dump to Google Drive. -# -# Authenticates with a Google Cloud service account key and uploads the -# database dump file to a Google Drive folder. A file with the same name in -# the folder is replaced in place, so its ID and sharing settings are kept. -# -# Service accounts have no storage quota of their own, so the folder must be -# in a shared drive. -# -# IMPORTANT! This script runs outside the container on the host system. -# -# shellcheck disable=SC1090,SC1091 - -t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ]; then . ./.env.local; fi && set +a && . "${t}" && rm "${t}" && unset t - -set -eu -set -o pipefail - -# Google Cloud service account key: the JSON content or a path to the JSON -# key file. -VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY:-${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY:-}}" - -# Google Drive folder ID to push the database dump file to. -# -# Defaults to the folder the database is fetched from. -VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID:-${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID:-}}" - -# Directory with database dump file. -VORTEX_PUSH_DB_GDRIVE_DB_DIR="${VORTEX_PUSH_DB_GDRIVE_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" - -# Database dump file name. -# -# Defaults to the name the database is fetched and refreshed with. The same -# name is used for the file in the Google Drive folder. -VORTEX_PUSH_DB_GDRIVE_DB_FILE="${VORTEX_PUSH_DB_GDRIVE_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" - -# ------------------------------------------------------------------------------ - -# @formatter:off -info() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[36m[INFO] %s\033[0m\n" "${1}" || printf "[INFO] %s\n" "${1}"; } -note() { printf " %s\n" "${1}"; } -task() { _TASK_START=$(date +%s); [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[34m[TASK] %s\033[0m\n" "${1}" || printf "[TASK] %s\n" "${1}"; } -pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START))s)" && unset _TASK_START; [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[32m[ OK ] %s%s\033[0m\n" "${1}" "${_d}" || printf "[ OK ] %s%s\n" "${1}" "${_d}"; } -fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } -# @formatter:on - -extract_json_last_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); \$last=array_pop(\$data); isset(\$last[\"${key}\"]) ? print trim(json_encode(\$last[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -extract_json_value() { - local key="${1}" - php -r "\$data=json_decode(file_get_contents('php://stdin'), TRUE); isset(\$data[\"${key}\"]) ? print trim(json_encode(\$data[\"${key}\"], JSON_UNESCAPED_SLASHES), '\"') : exit(1);" -} - -# Print a Google OAuth access token for a service account key and a scope. -# -# The key is passed to PHP in the environment, so it never appears in the -# process list. -gdrive_access_token() { - local key="${1}" - local scope="${2}" - - [ -f "${key}" ] && key="$(cat "${key}")" - - local assertion - # shellcheck disable=SC2016 - assertion=$(GDRIVE_KEY="${key}" GDRIVE_SCOPE="${scope}" php -r '$k=json_decode((string) getenv("GDRIVE_KEY"), TRUE); if (!is_array($k) || empty($k["client_email"]) || empty($k["private_key"])) { exit(1); } $b=fn($s) => rtrim(strtr(base64_encode($s), "+/", "-_"), "="); $t=time(); $u=$b(json_encode(["alg" => "RS256", "typ" => "JWT"])) . "." . $b(json_encode(["iss" => $k["client_email"], "scope" => getenv("GDRIVE_SCOPE"), "aud" => "https://oauth2.googleapis.com/token", "iat" => $t, "exp" => $t + 3600])); if (!@openssl_sign($u, $s, $k["private_key"], OPENSSL_ALGO_SHA256)) { exit(1); } print $u . "." . $b($s);') || return 1 - - local token_json - token_json=$(curl -sS -L https://oauth2.googleapis.com/token --data-urlencode "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" --data-urlencode "assertion=${assertion}") || return 1 - - echo "${token_json}" | extract_json_value "access_token" -} - -# Print the ID of the most recently modified file with a name in a folder. -# -# Prints nothing when the folder has no such file, and fails when the folder -# cannot be listed. -gdrive_find_file() { - local token="${1}" - local folder_id="${2}" - local name="${3}" - - local files_json - files_json=$(curl -sS -G https://www.googleapis.com/drive/v3/files -H "Authorization: Bearer ${token}" --data-urlencode "q=name = '${name//\'/\\\'}' and '${folder_id}' in parents and trashed = false" --data-urlencode "fields=files(id)" --data-urlencode "orderBy=modifiedTime desc" --data-urlencode "pageSize=1" --data-urlencode "supportsAllDrives=true" --data-urlencode "includeItemsFromAllDrives=true") || return 1 - - local files - files=$(echo "${files_json}" | extract_json_value "files") || return 1 - - echo "${files}" | extract_json_last_value "id" || true -} - -for cmd in php curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done - -[ -z "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." -[ -z "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" ] && fail "Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - -local_file="${VORTEX_PUSH_DB_GDRIVE_DB_DIR}/${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" -[ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." - -info "Started database dump push to Google Drive." - -note "Local file: ${local_file}" -note "Remote file: ${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" -note "Folder ID: ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" - -task "Authenticating with the service account." -token=$(gdrive_access_token "${VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY}" "https://www.googleapis.com/auth/drive") || fail "Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." -pass "Authenticated with the service account." - -file_id=$(gdrive_find_file "${token}" "${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" "${VORTEX_PUSH_DB_GDRIVE_DB_FILE}") || fail "Unable to list the ${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID} folder. Check that the folder ID is correct and that the folder is shared with the service account." - -upload_url="https://www.googleapis.com/upload/drive/v3/files" -if [ -n "${file_id}" ]; then - note "Replacing the existing file with ID ${file_id}." - request_type="PATCH" - upload_url="${upload_url}/${file_id}" - metadata="{}" -else - note "Creating a new file in the folder." - request_type="POST" - metadata=$(GDRIVE_NAME="${VORTEX_PUSH_DB_GDRIVE_DB_FILE}" GDRIVE_FOLDER_ID="${VORTEX_PUSH_DB_GDRIVE_FOLDER_ID}" php -r 'print json_encode(["name" => getenv("GDRIVE_NAME"), "parents" => [getenv("GDRIVE_FOLDER_ID")]]);') -fi - -# A resumable upload session accepts the whole file in a single request, so -# the dump is not limited by the 5 MB multipart upload size. -task "Starting the upload session." -session_headers=$(curl -sS -X "${request_type}" -D - -o /dev/null -H "Authorization: Bearer ${token}" -H "Content-Type: application/json; charset=UTF-8" -H "X-Upload-Content-Type: application/octet-stream" --data "${metadata}" "${upload_url}?uploadType=resumable&supportsAllDrives=true") || fail "Unable to start the upload session." -session_url=$(echo "${session_headers}" | sed -n 's/^[Ll]ocation: *//p' | tr -d '\r') -[ -z "${session_url}" ] && fail "Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." -pass "Started the upload session." - -task "Uploading the database dump file." -set +e -response=$(curl -sS -X PUT --upload-file "${local_file}" -H "Content-Type: application/octet-stream" --write-out "\n%{http_code}" "${session_url}") -curl_status=$? -set -e - -# Enable xtrace only after the authenticated requests so the access token and -# the upload session URL are never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - -[ "${curl_status}" -ne 0 ] && fail "Google Drive upload request failed with curl exit ${curl_status}." - -http_code=$(echo "${response}" | tail -1) -response_body=$(echo "${response}" | sed '$d') - -case "${http_code}" in - 2*) ;; - *) - [ -n "${response_body}" ] && echo "Response: ${response_body}" - fail "Unable to push to Google Drive: HTTP status ${http_code}." - ;; -esac -pass "Uploaded the database dump file." - -pass "Finished database dump push to Google Drive." diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index fd6986b1ff..4463c14f4f 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -4,6 +4,14 @@ # # Uses AWS Signature Version 4 with curl (no AWS CLI required). # +# Keeps 2 copies of the database dump file in the bucket: the current dump +# under the remote file name and the dump it replaced with a '-previous' +# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the +# current dump under the remote file name. +# +# The current dump is copied to the previous one on the S3 side before the +# upload, so a failed upload keeps the current dump. +# # IMPORTANT! This script runs outside the container on the host system. # # shellcheck disable=SC1090,SC1091 @@ -13,41 +21,69 @@ t=$(mktemp) && export -p >"${t}" && set -a && . ./.env && if [ -f ./.env.local ] set -eu set -o pipefail +# Database index suffix. When set (e.g., "2"), all DB-related variable lookups +# use the indexed variant, so each database pushes its own dump file. +_db_index="${VORTEX_DB_INDEX:-}" + # The S3 values fall back to the values the database is fetched with, so a # bucket used as the fetch source is configured once. # AWS access key. -VORTEX_PUSH_DB_S3_ACCESS_KEY="${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${VORTEX_FETCH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_ACCESS_KEY" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${!_v:-${!_vs:-${S3_ACCESS_KEY:-}}}" # AWS secret key. -VORTEX_PUSH_DB_S3_SECRET_KEY="${VORTEX_PUSH_DB_S3_SECRET_KEY:-${VORTEX_FETCH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY" +_vs="VORTEX_FETCH_DB${_db_index}_S3_SECRET_KEY" +VORTEX_PUSH_DB_S3_SECRET_KEY="${!_v:-${!_vs:-${S3_SECRET_KEY:-}}}" # S3 bucket name. -VORTEX_PUSH_DB_S3_BUCKET="${VORTEX_PUSH_DB_S3_BUCKET:-${VORTEX_FETCH_DB_S3_BUCKET:-${S3_BUCKET:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_BUCKET" +_vs="VORTEX_FETCH_DB${_db_index}_S3_BUCKET" +VORTEX_PUSH_DB_S3_BUCKET="${!_v:-${!_vs:-${S3_BUCKET:-}}}" # S3 region. -VORTEX_PUSH_DB_S3_REGION="${VORTEX_PUSH_DB_S3_REGION:-${VORTEX_FETCH_DB_S3_REGION:-${S3_REGION:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_REGION" +_vs="VORTEX_FETCH_DB${_db_index}_S3_REGION" +VORTEX_PUSH_DB_S3_REGION="${!_v:-${!_vs:-${S3_REGION:-}}}" # S3 prefix (path within the bucket). -VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX:-${VORTEX_FETCH_DB_S3_PREFIX:-${S3_PREFIX:-}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_PREFIX" +_vs="VORTEX_FETCH_DB${_db_index}_S3_PREFIX" +VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" # Directory with database dump file. -VORTEX_PUSH_DB_S3_DB_DIR="${VORTEX_PUSH_DB_S3_DB_DIR:-${VORTEX_DB_DIR:-./.data}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" +_vs="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${VORTEX_DB_DIR:-./.data}}}" # Database dump file name. # # Defaults to the name the database is fetched and refreshed with. -VORTEX_PUSH_DB_S3_DB_FILE="${VORTEX_PUSH_DB_S3_DB_FILE:-${VORTEX_FETCH_DB_FILE:-${VORTEX_DB_FILE:-db.sql}}}" +_v="VORTEX_PUSH_DB${_db_index}_S3_DB_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_FILE" +_vss="VORTEX_DB${_db_index}_FILE" +VORTEX_PUSH_DB_S3_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" # Remote database dump file name. # -# Defaults to the database dump file name, which is the name the database is -# fetched with. -VORTEX_PUSH_DB_S3_REMOTE_FILE="${VORTEX_PUSH_DB_S3_REMOTE_FILE:-${VORTEX_PUSH_DB_S3_DB_FILE}}" +# Defaults to the remote file name the database is fetched from, then to the +# database dump file name. +_v="VORTEX_PUSH_DB${_db_index}_S3_REMOTE_FILE" +_vs="VORTEX_FETCH_DB${_db_index}_S3_DB_FILE" +VORTEX_PUSH_DB_S3_REMOTE_FILE="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_DB_FILE}}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" +# Prefix for the names of the pushed database dump files. +# +# Use a different prefix in each place that pushes to the same storage, such +# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the +# prefixed name only when its remote file name includes the prefix. +VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" + # ------------------------------------------------------------------------------ # @formatter:off @@ -94,37 +130,123 @@ aws_signature() { hmac_sha256 "hexkey:${signing_key}" "${string_to_sign}" } +# Send a signed PUT request for an object and print the response body +# followed by the HTTP status code on the last line. +# +# Arguments: the encoded object key, the payload hash, the file to upload +# (empty for none), then any extra 'name:value' headers to sign, with +# lowercase names. +s3_put() { + local key="${1}" + local payload_hash="${2}" + local upload_file="${3}" + shift 3 + + local host="${VORTEX_PUSH_DB_S3_BUCKET}.s3.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" + local date_short date_long + date_short=$(date -u +'%Y%m%d') + date_long=$(date -u +'%Y%m%dT%H%M%SZ') + + local canonical_headers signed_headers + canonical_headers=$(printf '%s\n' "host:${host}" "x-amz-content-sha256:${payload_hash}" "x-amz-date:${date_long}" "x-amz-server-side-encryption:AES256" "x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" "$@" | LC_ALL=C sort) + signed_headers=$(echo "${canonical_headers}" | cut -d: -f1 | paste -sd ';' -) + + local canonical_request="PUT +/${key} + +${canonical_headers} + +${signed_headers} +${payload_hash}" + + local canonical_request_hash + canonical_request_hash=$(hash_sha256 "${canonical_request}") + + local string_to_sign="AWS4-HMAC-SHA256 +${date_long} +${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request +${canonical_request_hash}" + + local signature + signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "s3" "${string_to_sign}") + + local args=(--silent --show-error --location --proto-redir "=https" --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" --request PUT --write-out "\n%{http_code}") + if [ -n "${upload_file}" ]; then + args+=(--upload-file "${upload_file}") + fi + + local header + while IFS= read -r header; do + args+=(--header "${header%%:*}: ${header#*:}") + done <<<"${canonical_headers}" + args+=(--header "Authorization: AWS4-HMAC-SHA256 Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/s3/aws4_request, SignedHeaders=${signed_headers}, Signature=${signature}") + + curl "${args[@]}" "https://${host}/${key}" +} + for cmd in curl openssl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done -[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_ACCESS_KEY." -[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_SECRET_KEY." -[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_BUCKET." -[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB_S3_REGION." +[ -z "${VORTEX_PUSH_DB_S3_ACCESS_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY." +[ -z "${VORTEX_PUSH_DB_S3_SECRET_KEY}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY." +[ -z "${VORTEX_PUSH_DB_S3_BUCKET}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_BUCKET." +[ -z "${VORTEX_PUSH_DB_S3_REGION}" ] && fail "Missing required value for VORTEX_PUSH_DB${_db_index}_S3_REGION." local_file="${VORTEX_PUSH_DB_S3_DB_DIR}/${VORTEX_PUSH_DB_S3_DB_FILE}" [ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." -info "Started database dump push to S3." +info "Started database${_db_index:+ ${_db_index}} dump push to S3." [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX%/}/" -request_type="PUT" -auth_type="AWS4-HMAC-SHA256" -service="s3" -base_url=".${service}.${VORTEX_PUSH_DB_S3_REGION}.amazonaws.com" -date_short=$(date -u +'%Y%m%d') -date_long=$(date -u +'%Y%m%dT%H%M%SZ') +# The file name prefix and the '-previous' suffix apply to the file name, not +# to the directories in the remote file name. +remote_dir="" +remote_name="${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +if [[ ${remote_name} == */* ]]; then + remote_dir="${remote_name%/*}/" + remote_name="${remote_name##*/}" +fi + +current_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +if [[ ${remote_name} == *.* ]]; then + previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" +else + previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" +fi -object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}")" -object_url="https://${VORTEX_PUSH_DB_S3_BUCKET}${base_url}/${object_key}" +object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${current_file}")" +previous_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}")" note "Local file: ${local_file}" -note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${VORTEX_PUSH_DB_S3_REMOTE_FILE}" +note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${current_file}" +note "Previous file: ${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}" note "S3 bucket: ${VORTEX_PUSH_DB_S3_BUCKET}" note "S3 region: ${VORTEX_PUSH_DB_S3_REGION}" [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && note "S3 prefix: ${VORTEX_PUSH_DB_S3_PREFIX}" note "Storage class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" +# A copy can fail after S3 has sent a 200 status, so a success needs the copy +# result in the body. A missing current dump means this is the first push. +task "Keeping the current database dump as the previous one." +set +e +response=$(s3_put "${previous_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") +curl_status=$? +set -e + +[ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." + +http_code=$(echo "${response}" | tail -1) +response_body=$(echo "${response}" | sed '$d') + +if [[ ${http_code} == 2* ]] && [[ ${response_body} == *"/dev/null; then content_type="$(file --brief --mime-type "${local_file}")" else @@ -133,51 +255,14 @@ fi payload_hash=$(openssl dgst -sha256 <"${local_file}" | sed 's/^.* //') -header_list='content-type;host;x-amz-content-sha256;x-amz-date;x-amz-server-side-encryption;x-amz-storage-class' - -canonical_request="\ -${request_type} -/${object_key} - -content-type:${content_type} -host:${VORTEX_PUSH_DB_S3_BUCKET}${base_url} -x-amz-content-sha256:${payload_hash} -x-amz-date:${date_long} -x-amz-server-side-encryption:AES256 -x-amz-storage-class:${VORTEX_PUSH_DB_S3_STORAGE_CLASS} - -${header_list} -${payload_hash}" - -canonical_request_hash=$(hash_sha256 "${canonical_request}") - -string_to_sign="\ -${auth_type} -${date_long} -${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request -${canonical_request_hash}" - -signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "${service}" "${string_to_sign}") - +task "Uploading the database dump file." set +e -response=$(curl --silent --show-error --location --proto-redir =https \ - --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" \ - --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" \ - --request "${request_type}" --upload-file "${local_file}" \ - --header "Content-Type: ${content_type}" \ - --header "Host: ${VORTEX_PUSH_DB_S3_BUCKET}${base_url}" \ - --header "X-Amz-Content-SHA256: ${payload_hash}" \ - --header "X-Amz-Date: ${date_long}" \ - --header "X-Amz-Server-Side-Encryption: AES256" \ - --header "X-Amz-Storage-Class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" \ - --header "Authorization: ${auth_type} Credential=${VORTEX_PUSH_DB_S3_ACCESS_KEY}/${date_short}/${VORTEX_PUSH_DB_S3_REGION}/${service}/aws4_request, SignedHeaders=${header_list}, Signature=${signature}" \ - --write-out "\n%{http_code}" \ - "${object_url}") +response=$(s3_put "${object_key}" "${payload_hash}" "${local_file}" "content-type:${content_type}") curl_status=$? set -e -# Enable xtrace only after the signed request so credentials and the -# signature are never written to the trace output. +# Enable xtrace only after the signed requests so credentials and the +# signatures are never written to the trace output. [ "${VORTEX_DEBUG-}" = "1" ] && set -x [ "${curl_status}" -ne 0 ] && fail "S3 push request failed with curl exit ${curl_status}." @@ -192,5 +277,6 @@ case "${http_code}" in fail "Unable to push to S3: HTTP status ${http_code}." ;; esac +pass "Uploaded the database dump file." -pass "Finished database dump push to S3." +pass "Finished database${_db_index:+ ${_db_index}} dump push to S3." diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index df899cafd5..f32cf74240 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,9 +15,10 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # -# With a dump store fetch source ('gdrive' or 's3') and VORTEX_PUSH_DB_PROCEED -# set to 1, the database is sanitized after provisioning, so the exported -# dump can be pushed to the store with 'vortex-push-db'. +# With an 'ftp' or 's3' fetch source that has an upstream source in +# VORTEX_REFRESH_DB_SOURCE (a dump store) and VORTEX_PUSH_DB_PROCEED set to 1, +# the database is sanitized after provisioning, so the exported dump can be +# pushed to the store with 'vortex-push-db'. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -47,12 +48,13 @@ VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" # # Required when the fetch source is 'container_registry' and a base image is # set: the database image is built from a dump file, so it needs a dump file -# source such as 'url', 'ftp', 'acquia', 'lagoon', 's3' or 'gdrive'. An -# indexed database uses VORTEX_REFRESH_DB2_SOURCE. +# source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed +# database uses VORTEX_REFRESH_DB2_SOURCE. # -# Also required when the fetch source is a dump store ('gdrive' or 's3') and -# VORTEX_PUSH_DB_PROCEED is set to 1: 'vortex-fetch-db' then fetches the dump -# from this source, such as 'url', 'ftp', 'acquia' or 'lagoon'. +# With an 'ftp' or 's3' fetch source, setting this value makes the fetch +# source a dump store: with VORTEX_PUSH_DB_PROCEED set to 1, 'vortex-fetch-db' +# fetches the dump from this source and 'vortex-push-db' pushes the exported +# dump to the store. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -106,12 +108,12 @@ fi rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 -# A dump store ('gdrive' or 's3') receives the dump this refresh exports, so -# with a push requested the exported dump must be sanitized. +# A dump store receives the dump this refresh exports, so with a push +# requested the exported dump must be sanitized. refresh_store=0 -if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then +if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then case "${fetch_source}" in - gdrive | s3) refresh_store=1 ;; + ftp | s3) refresh_store=1 ;; esac fi @@ -121,7 +123,7 @@ fi if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then case "${VORTEX_REFRESH_DB_SOURCE}" in - "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." ;; + "" | container_registry | none) fail "The ${db_image} image is built from a database dump file, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${VORTEX_REFRESH_DB_SOURCE}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." ;; esac fi diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 61bb8a2d46..a22c442003 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -122,6 +122,7 @@ setup() { unset VORTEX_FETCH_DB_FORCE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED unset VORTEX_PUSH_DB_PROCEED + unset VORTEX_PUSH_DB_FILE_PREFIX # Isolate ambient credentials from the scripts under test. Scripts fall back # to these unprefixed values whenever their own variable is unset or empty, diff --git a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats b/.vortex/tooling/tests/unit/fetch-db-gdrive.bats deleted file mode 100644 index 987005923e..0000000000 --- a/.vortex/tooling/tests/unit/fetch-db-gdrive.bats +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env bats -## -# Unit tests for the 'vortex-fetch-db-gdrive' script. -# -# shellcheck disable=SC2016,SC2030,SC2031,SC2034 - -load ../_helper.bash - -# Writes a service account key file with a newly generated private key. -fixture_gdrive_key() { - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null - php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" -} - -@test "fetch-db-gdrive: Fetch database file successfully with a key file" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token","expires_in":3599}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_side_effect "${mock_curl}" "echo 'CREATE TABLE test (id INT);' > .data/db.sql" 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - assert_output_contains "[INFO] Started database dump fetch from Google Drive." - assert_output_contains "Remote file: db.sql" - assert_output_contains "Local path: .data/db.sql" - assert_output_contains "Folder ID: folder-123" - assert_output_contains "[ OK ] Authenticated with the service account." - assert_output_contains "[ OK ] Found the database dump file with ID file-456." - assert_output_contains "[ OK ] Downloaded the database dump file." - assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." - assert_output_not_contains "test-token" - - assert_equal "3" "$(mock_get_call_num "${mock_curl}")" - - # The token request carries a signed JWT assertion for the read-only scope. - token_args="$(mock_get_call_args "${mock_curl}" 1)" - assert_string_contains "${token_args}" "https://oauth2.googleapis.com/token" - assert_string_contains "${token_args}" "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer" - assertion="${token_args##*assertion=}" - assertion="${assertion%% *}" - IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" - assert_not_empty "${jwt_signature}" - claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" - assert_string_contains "${claims}" '"iss":"vortex@example.iam.gserviceaccount.com"' - assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive.readonly"' - - list_args="$(mock_get_call_args "${mock_curl}" 2)" - assert_string_contains "${list_args}" "https://www.googleapis.com/drive/v3/files" - assert_string_contains "${list_args}" "Authorization: Bearer test-token" - assert_string_contains "${list_args}" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" - assert_string_contains "${list_args}" "supportsAllDrives=true" - - download_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${download_args}" "Authorization: Bearer test-token" - assert_string_contains "${download_args}" "-o .data/db.sql" - assert_string_contains "${download_args}" "https://www.googleapis.com/drive/v3/files/file-456?alt=media&supportsAllDrives=true" - - assert_file_contains ".data/db.sql" "CREATE TABLE test" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fetch database file with the key content in the shared variable" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - unset VORTEX_FETCH_DB_GDRIVE_DB_DIR VORTEX_FETCH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - # The default directory and file name come from .env. - assert_output_contains "Local path: ./.data/db.sql" - assert_output_contains "[ OK ] Finished database dump fetch from Google Drive." - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 3)" "-o ./.data/db.sql" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Resolve indexed variables with VORTEX_DB_INDEX" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_DB_INDEX=3 - export VORTEX_FETCH_DB3_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB3_GDRIVE_FOLDER_ID="folder-789" - export VORTEX_FETCH_DB3_FILE="db3.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-3"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - assert_output_contains "Remote file: db3.sql" - assert_output_contains "Folder ID: folder-789" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db3.sql' and 'folder-789' in parents and trashed = false" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Escape a quote in the file name of the folder query" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="site's.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 0 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_success - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'site\\'s.sql' and" - - rm -rf .data - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the service account key is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder ID is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the service account key is invalid" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - for key in "not json" '{"client_email":"vortex@example.iam.gserviceaccount.com"}' '{"client_email":"vortex@example.iam.gserviceaccount.com","private_key":"not a key"}'; do - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${key}" - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." - done - - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the token request is rejected" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"error":"invalid_grant","error_description":"Invalid JWT Signature."}' 1 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account." - assert_output_not_contains "Found the database dump file" - assert_equal "1" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder cannot be listed" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"error":{"code":404,"message":"File not found: folder-123."}}' 2 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the folder has no database dump file" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to find the db.sql file in the folder-123 folder." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "fetch-db-gdrive: Fail when the download fails" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - fixture_gdrive_key - - export VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_status "${mock_curl}" 22 3 - - run .vortex/tooling/src/vortex-fetch-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to download the database dump file." - assert_output_not_contains "Finished database dump fetch from Google Drive." - - rm -rf .data - - popd >/dev/null -} diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index bd8d08e665..0ab820b840 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -14,7 +14,6 @@ stub_fetch_scripts() { stub_sibling "vortex-fetch-db-lagoon" "Fetched from lagoon." stub_sibling "vortex-fetch-db-container-registry" "Fetched from container_registry." stub_sibling "vortex-fetch-db-s3" "Fetched from s3." - stub_sibling "vortex-fetch-db-gdrive" "Fetched from gdrive." } @test "fetch-db: Fetch with URL source" { @@ -308,7 +307,7 @@ EOF export VORTEX_FETCH_DB_PROCEED="1" - for db_source in url ftp acquia lagoon container_registry s3 gdrive; do + for db_source in url ftp acquia lagoon container_registry s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -316,7 +315,7 @@ EOF assert_output_contains "Fetched from ${db_source}." assert_output_contains "Finished database fetch." - for other_source in url ftp acquia lagoon container_registry s3 gdrive; do + for other_source in url ftp acquia lagoon container_registry s3; do if [ "${other_source}" != "${db_source}" ]; then assert_output_not_contains "Fetched from ${other_source}." fi @@ -335,7 +334,7 @@ EOF export VORTEX_PUSH_DB_PROCEED="1" export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in gdrive s3; do + for db_source in ftp s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db @@ -349,62 +348,82 @@ EOF popd >/dev/null } +@test "fetch-db: Fetch an indexed dump store source from its refresh source with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_DB_INDEX=3 + export VORTEX_FETCH_DB3_SOURCE="ftp" + export VORTEX_FETCH_DB3_PROCEED="1" + export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_PUSH_DB_PROCEED="1" + export VORTEX_REFRESH_DB3_SOURCE="url" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the url source in VORTEX_REFRESH_DB3_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetched from url." + assert_output_not_contains "Fetched from ftp." + assert_output_not_contains "Fetched from lagoon." + + popd >/dev/null +} + @test "fetch-db: Fetch a dump store source from the store without a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_REFRESH_DB_SOURCE="lagoon" unset VORTEX_PUSH_DB_PROCEED run .vortex/tooling/src/vortex-fetch-db assert_success - assert_output_contains "Fetched from gdrive." + assert_output_contains "Fetched from ftp." assert_output_not_contains "Fetched from lagoon." popd >/dev/null } -@test "fetch-db: Fetch an indexed dump store source from the store with a push requested" { +@test "fetch-db: Fetch an ftp source without a refresh source as an upstream source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_DB_INDEX=3 - export VORTEX_FETCH_DB3_SOURCE="gdrive" - export VORTEX_FETCH_DB3_PROCEED="1" - export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_REFRESH_DB_SOURCE="" run .vortex/tooling/src/vortex-fetch-db assert_success - # Only the primary database is pushed to a dump store. - assert_output_contains "Fetched from gdrive." - assert_output_not_contains "Fetched from lagoon." + assert_output_not_contains "Fetching from the" + assert_output_contains "Fetched from ftp." popd >/dev/null } -@test "fetch-db: Fail without an upstream source for a dump store source with a push requested" { +@test "fetch-db: Fail with an invalid upstream source for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" - for refresh_source in "" gdrive s3 container_registry none; do + for refresh_source in ftp container_registry none; do export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" run .vortex/tooling/src/vortex-fetch-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia or lagoon." - assert_output_not_contains "Fetched from gdrive." + assert_output_contains "[FAIL] The database dump file pushed to ftp is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_not_contains "Fetched from ftp." assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" done @@ -444,7 +463,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database fetch." - assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' + assert_output_contains 'Unsupported database source "VORTEX_CONTAINER_REGISTRY" in VORTEX_FETCH_DB_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' assert_output_not_contains "Fetched from" assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" @@ -484,7 +503,7 @@ EOF run .vortex/tooling/src/vortex-fetch-db assert_failure_status 1 assert_output_contains "Started database 2 fetch." - assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, gdrive, none.' + assert_output_contains 'Unsupported database source "sftp" in VORTEX_FETCH_DB2_SOURCE. Use one of: url, ftp, acquia, lagoon, container_registry, s3, none.' assert_output_not_contains "Finished database 2 fetch." popd >/dev/null diff --git a/.vortex/tooling/tests/unit/push-db-ftp.bats b/.vortex/tooling/tests/unit/push-db-ftp.bats new file mode 100644 index 0000000000..b94fc182bf --- /dev/null +++ b/.vortex/tooling/tests/unit/push-db-ftp.bats @@ -0,0 +1,204 @@ +#!/usr/bin/env bats +## +# Unit tests for the 'vortex-push-db-ftp' script. +# +# shellcheck disable=SC2030,SC2031,SC2034 + +load ../_helper.bash + +# Exports the FTP values the database is fetched with. +setup_ftp_fetch_values() { + export VORTEX_FETCH_DB_FTP_USER="fetch-user" + export VORTEX_FETCH_DB_FTP_PASS="fetch-pass" + export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB_FTP_PORT="21" + export VORTEX_FETCH_DB_FTP_FILE="db.sql" + unset VORTEX_PUSH_DB_FTP_USER VORTEX_PUSH_DB_FTP_PASS VORTEX_PUSH_DB_FTP_HOST VORTEX_PUSH_DB_FTP_PORT VORTEX_PUSH_DB_FTP_FILE VORTEX_PUSH_DB_FILE_PREFIX +} + +@test "push-db-ftp: Upload the dump and keep the dump it replaces" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "CREATE TABLE test (id INT);" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database dump push to FTP." + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "Previous file: db-previous.sql" + assert_output_contains "FTP host: ftp.example.com:21" + assert_output_contains "[ OK ] Uploaded the database dump file and kept the dump it replaced." + assert_output_contains "[ OK ] Finished database dump push to FTP." + assert_output_not_contains "fetch-pass" + + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--config - --ftp-create-dirs --upload-file ./.data/db.sql" + # The upload takes a temporary name, and the rename into place comes last. + assert_string_contains "${args}" "--quote -*DELE db-previous.sql --quote -*RNFR db.sql --quote -*RNTO db-previous.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" + assert_string_contains "${args}" "ftp://ftp.example.com:21/db.sql.uploading" + # The credentials reach curl on stdin, never as arguments. + assert_string_not_contains "${args}" "fetch-pass" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:fetch-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to a remote directory with a file name prefix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_FTP_HOST="push.example.com" + export VORTEX_PUSH_DB_FTP_PORT="2121" + export VORTEX_PUSH_DB_FTP_FILE="/backups/site/db.sql" + export VORTEX_PUSH_DB_FILE_PREFIX="gha-" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: backups/site/gha-db.sql" + assert_output_contains "Previous file: backups/site/gha-db-previous.sql" + assert_output_contains "FTP host: push.example.com:2121" + + args="$(mock_get_call_args "${mock_curl}" 1)" + # The rename commands run in the remote directory, so they use bare names. + assert_string_contains "${args}" "--quote -*DELE gha-db-previous.sql --quote -*RNFR gha-db.sql --quote -*RNTO gha-db-previous.sql --quote -RNFR gha-db.sql.uploading --quote -RNTO gha-db.sql" + assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/gha-db.sql.uploading" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push an indexed database with its own variables and file" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + setup_ftp_fetch_values + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_FTP_USER="db2-user" + export VORTEX_FETCH_DB2_FTP_PASS="db2-pass" + export VORTEX_FETCH_DB2_FTP_HOST="ftp2.example.com" + export VORTEX_FETCH_DB2_FTP_PORT="21" + export VORTEX_FETCH_DB2_FTP_FILE="db2.sql" + export VORTEX_FETCH_DB2_FILE="db2.sql" + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[INFO] Started database 2 dump push to FTP." + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: db2.sql" + assert_output_contains "Previous file: db2-previous.sql" + assert_output_contains "[ OK ] Finished database 2 dump push to FTP." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp2.example.com:21/db2.sql.uploading" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "db2-user:db2-pass"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Escape quotes and backslashes in the credentials" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_FETCH_DB_FTP_PASS='pa"ss\word' + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "fetch-user:pa\"ss\\word"' + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when a required value is missing" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + mock_curl=$(mock_command "curl") + + for name in USER PASS HOST FILE; do + setup_ftp_fetch_values + export "VORTEX_FETCH_DB_FTP_${name}=" + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_FTP_${name} or VORTEX_FETCH_DB_FTP_${name}." + done + + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the database dump file does not exist" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + rm -rf .data + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Database dump file ./.data/db.sql does not exist." + assert_output_not_contains "Started database dump push to FTP." + assert_equal "0" "$(mock_get_call_num "${mock_curl}")" + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the upload or a rename fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 21 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[FAIL] Unable to push the database dump file to FTP." + assert_output_not_contains "Finished database dump push to FTP." + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-gdrive.bats b/.vortex/tooling/tests/unit/push-db-gdrive.bats deleted file mode 100644 index 35773af729..0000000000 --- a/.vortex/tooling/tests/unit/push-db-gdrive.bats +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bats -## -# Unit tests for the 'vortex-push-db-gdrive' script. -# -# shellcheck disable=SC2016,SC2030,SC2031,SC2034 - -load ../_helper.bash - -# Writes a service account key file with a newly generated private key. -fixture_gdrive_key() { - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${BUILD_DIR}/gdrive.pem" 2>/dev/null - php -r 'print json_encode(["type" => "service_account", "client_email" => "vortex@example.iam.gserviceaccount.com", "private_key" => file_get_contents($argv[1])]);' "${BUILD_DIR}/gdrive.pem" >"${BUILD_DIR}/gdrive.json" -} - -@test "push-db-gdrive: Create the database dump file in the folder" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "CREATE TABLE test (id INT);" >.data/db.sql - fixture_gdrive_key - - unset VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1\r\ncontent-length: 0\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "[INFO] Started database dump push to Google Drive." - assert_output_contains "Local file: .data/db.sql" - assert_output_contains "Remote file: db.sql" - assert_output_contains "Folder ID: folder-123" - assert_output_contains "[ OK ] Authenticated with the service account." - assert_output_contains "Creating a new file in the folder." - assert_output_contains "[ OK ] Started the upload session." - assert_output_contains "[ OK ] Uploaded the database dump file." - assert_output_contains "[ OK ] Finished database dump push to Google Drive." - assert_output_not_contains "test-token" - assert_output_not_contains "session-1" - - assert_equal "4" "$(mock_get_call_num "${mock_curl}")" - - # The token request carries a signed JWT assertion for the full Drive scope. - token_args="$(mock_get_call_args "${mock_curl}" 1)" - assertion="${token_args##*assertion=}" - assertion="${assertion%% *}" - IFS='.' read -r jwt_header jwt_claims jwt_signature <<<"${assertion}" - assert_not_empty "${jwt_signature}" - claims="$(php -r 'print base64_decode(strtr($argv[1], "-_", "+/"));' "${jwt_claims}")" - assert_string_contains "${claims}" '"scope":"https:\/\/www.googleapis.com\/auth\/drive"' - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db.sql' and 'folder-123' in parents and trashed = false" - - session_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${session_args}" "-X POST" - assert_string_contains "${session_args}" "Authorization: Bearer test-token" - assert_string_contains "${session_args}" '--data {"name":"db.sql","parents":["folder-123"]}' - assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" - - upload_args="$(mock_get_call_args "${mock_curl}" 4)" - assert_string_contains "${upload_args}" "-X PUT --upload-file .data/db.sql" - assert_string_contains "${upload_args}" "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&upload_id=session-1" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Replace the existing database dump file in place" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="$(cat "${BUILD_DIR}/gdrive.json")" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY - # The folder falls back to the folder the database is fetched from. - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="fetch-folder" - unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[{"id":"file-456"}]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nLocation: https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&upload_id=session-2\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-456"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "Local file: ./.data/db.sql" - assert_output_contains "Folder ID: fetch-folder" - assert_output_contains "Replacing the existing file with ID file-456." - assert_output_contains "[ OK ] Finished database dump push to Google Drive." - - session_args="$(mock_get_call_args "${mock_curl}" 3)" - assert_string_contains "${session_args}" "-X PATCH" - assert_string_contains "${session_args}" "--data {}" - assert_string_contains "${session_args}" "https://www.googleapis.com/upload/drive/v3/files/file-456?uploadType=resumable&supportsAllDrives=true" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "upload_id=session-2" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the service account key is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY="" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the folder ID is missing" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="" - export VORTEX_FETCH_DB_GDRIVE_FOLDER_ID="" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Missing required value for VORTEX_PUSH_DB_GDRIVE_FOLDER_ID or VORTEX_FETCH_DB_GDRIVE_FOLDER_ID." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the database dump file does not exist" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - rm -rf .data - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="{}" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - export VORTEX_PUSH_DB_GDRIVE_DB_FILE="db.sql" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Database dump file .data/db.sql does not exist." - assert_output_not_contains "Started database dump push to Google Drive." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the service account key is invalid" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="not json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to authenticate with the service account. Check that VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY or VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY is a valid service account JSON key." - assert_equal "0" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail without creating a file when the folder cannot be listed" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"error":{"code":403,"message":"Insufficient permissions."}}' 2 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to list the folder-123 folder. Check that the folder ID is correct and that the folder is shared with the service account." - assert_output_not_contains "Creating a new file in the folder." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload session has no location" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 403\r\ncontent-type: application/json\r\n' 3 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Unable to start the upload session: the response has no upload location. Check that the folder is in a shared drive and that the service account can edit it." - assert_equal "3" "$(mock_get_call_num "${mock_curl}")" - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload returns a non-2xx status" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-3\r\n' 3 - mock_set_output "${mock_curl}" $'{"error":{"code":403,"message":"The user does not have sufficient permissions."}}\n403' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "The user does not have sufficient permissions." - assert_output_contains "[FAIL] Unable to push to Google Drive: HTTP status 403." - assert_output_not_contains "Finished database dump push to Google Drive." - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Fail when the upload request fails" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_PUSH_DB_GDRIVE_DB_DIR=".data" - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-4\r\n' 3 - mock_set_status "${mock_curl}" 28 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_failure - assert_output_contains "[FAIL] Google Drive upload request failed with curl exit 28." - - rm -rf .data - - popd >/dev/null -} - -@test "push-db-gdrive: Push the database dump file the database is fetched with" { - pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - - mkdir -p .data - echo "database content" >.data/db_fetched.sql - fixture_gdrive_key - - export VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY="${BUILD_DIR}/gdrive.json" - export VORTEX_PUSH_DB_GDRIVE_FOLDER_ID="folder-123" - export VORTEX_FETCH_DB_FILE="db_fetched.sql" - unset VORTEX_PUSH_DB_GDRIVE_DB_DIR VORTEX_PUSH_DB_GDRIVE_DB_FILE - - mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" '{"access_token":"test-token"}' 1 - mock_set_output "${mock_curl}" '{"files":[]}' 2 - mock_set_output "${mock_curl}" $'HTTP/2 200\r\nlocation: https://www.googleapis.com/upload/drive/v3/files?upload_id=session-5\r\n' 3 - mock_set_output "${mock_curl}" $'{"id":"file-new"}\n200' 4 - - run .vortex/tooling/src/vortex-push-db-gdrive - assert_success - assert_output_contains "Local file: ./.data/db_fetched.sql" - assert_output_contains "Remote file: db_fetched.sql" - - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "q=name = 'db_fetched.sql' and" - assert_string_contains "$(mock_get_call_args "${mock_curl}" 4)" "--upload-file ./.data/db_fetched.sql" - - rm -rf .data - - popd >/dev/null -} diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index c467452983..72793df610 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -21,6 +21,7 @@ load ../_helper.bash "S3 region: ap-southeast-2" "Storage class: STANDARD" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -61,6 +62,7 @@ load ../_helper.bash "Local file: ./.data/db.sql" "Remote file: db.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -100,6 +102,7 @@ load ../_helper.bash "S3 bucket: shortcut-bucket" "S3 region: us-east-1" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -143,6 +146,7 @@ load ../_helper.bash "Remote file: backup/db_latest.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -183,6 +187,7 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -223,6 +228,7 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -261,6 +267,7 @@ load ../_helper.bash declare -a STEPS=( "[INFO] Started database dump push to S3." + "@curl * # 0 # \n200" "@curl * # 0 # AccessDenied\n403" "[FAIL] Unable to push to S3: HTTP status 403." @@ -439,6 +446,7 @@ load ../_helper.bash "S3 region: eu-west-1" "S3 prefix: store/" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -484,6 +492,7 @@ load ../_helper.bash "Local file: ./.data/db_fetched.sql" "Remote file: db_fetched.sql" + "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -509,3 +518,139 @@ load ../_helper.bash popd >/dev/null } + +# Exports the variables of a push to the 'test-bucket' bucket with the +# 'backups/' prefix. +setup_s3_push() { + mkdir -p .data + echo "database content" >.data/db.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_FETCH_DB_S3_ACCESS_KEY="" VORTEX_FETCH_DB_S3_SECRET_KEY="" VORTEX_FETCH_DB_S3_BUCKET="" VORTEX_FETCH_DB_S3_REGION="" VORTEX_FETCH_DB_S3_PREFIX="" + + export VORTEX_PUSH_DB_S3_ACCESS_KEY="testaccesskey" + export VORTEX_PUSH_DB_S3_SECRET_KEY="testsecretkey" + export VORTEX_PUSH_DB_S3_BUCKET="test-bucket" + export VORTEX_PUSH_DB_S3_REGION="ap-southeast-2" + export VORTEX_PUSH_DB_S3_PREFIX="backups" + export VORTEX_PUSH_DB_S3_DB_DIR=".data" + export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" + unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_PREFIX +} + +@test "push-db-s3: Keep the current dump as the previous one before the upload" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'\n200' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "Remote file: backups/db.sql" + assert_output_contains "Previous file: backups/db-previous.sql" + assert_output_contains "[ OK ] Kept the current database dump as the previous one." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Finished database dump push to S3." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The copy names the current dump as its source and sends no body. + copy_args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${copy_args}" "--request PUT" + assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" + assert_string_not_contains "${copy_args}" "--upload-file" + assert_string_contains "${copy_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db-previous.sql" + + upload_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${upload_args}" "--upload-file .data/db.sql" + assert_string_contains "${upload_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Upload the first dump when there is no current one" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'NoSuchKey\n404' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[ OK ] Skipped keeping the previous database dump as there is no current one." + assert_output_contains "[ OK ] Finished database dump push to S3." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Fail without uploading when the current dump cannot be kept" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + setup_s3_push + + # A copy error can come with a 200 status. Each run makes 1 request. + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'InternalError\n200' 1 + mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 + + for status in 200 403; do + run .vortex/tooling/src/vortex-push-db-s3 + assert_failure + assert_output_contains "Response: " + assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${status}." + assert_output_not_contains "Uploaded the database dump file." + done + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-s3: Push an indexed database with a file name prefix" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_S3_ACCESS_KEY="db2-access-key" + export VORTEX_FETCH_DB2_S3_SECRET_KEY="db2-secret-key" + export VORTEX_FETCH_DB2_S3_BUCKET="db2-bucket" + export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" + export VORTEX_FETCH_DB2_S3_PREFIX="" + export VORTEX_FETCH_DB2_FILE="db2.sql" + export VORTEX_PUSH_DB_FILE_PREFIX="circleci-" + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" $'\n200' 1 + mock_set_output "${mock_curl}" "200" 2 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + assert_output_contains "[INFO] Started database 2 dump push to S3." + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: circleci-db2.sql" + assert_output_contains "Previous file: circleci-db2-previous.sql" + assert_output_contains "S3 bucket: db2-bucket" + assert_output_contains "[ OK ] Finished database 2 dump push to S3." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "--header x-amz-copy-source: /db2-bucket/circleci-db2.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/circleci-db2.sql" + + rm -rf .data + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats index d2dee534f0..dbc3d256f4 100644 --- a/.vortex/tooling/tests/unit/push-db.bats +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -2,16 +2,21 @@ ## # Unit tests for the 'vortex-push-db' script. # -# shellcheck disable=SC2030,SC2031 +# The stub markers use single quotes on purpose: the `${...}` must reach the +# generated stub literally and expand when the stub runs - so SC2016 is +# suppressed here. +# +# shellcheck disable=SC2016,SC2030,SC2031 load ../_helper.bash @test "push-db: Skip when the push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" for proceed in "" 0; do export VORTEX_PUSH_DB_PROCEED="${proceed}" @@ -20,7 +25,7 @@ load ../_helper.bash assert_success assert_output_contains "[INFO] Started database push." assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." - assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Pushed to ftp." assert_output_not_contains "Finished database push." done @@ -30,42 +35,66 @@ load ../_helper.bash @test "push-db: Route each dump store source to its push script" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." - stub_sibling "vortex-push-db-s3" "Pushed to s3." + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + stub_sibling "vortex-push-db-s3" 'Pushed to s3 with index [${VORTEX_DB_INDEX}].' export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in gdrive s3; do + for db_source in ftp s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "Pushed to ${db_source}." + assert_output_contains "Pushed to ${db_source} with index []." assert_output_contains "[ OK ] Finished database push." - for other_source in gdrive s3; do + for other_source in ftp s3; do [ "${other_source}" = "${db_source}" ] && continue - assert_output_not_contains "Pushed to ${other_source}." + assert_output_not_contains "Pushed to ${other_source}" done done popd >/dev/null } +@test "push-db: Push an indexed database with its own variables" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + + export VORTEX_DB_INDEX=2 + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB2_SOURCE="ftp" + export VORTEX_REFRESH_DB2_SOURCE="url" + # The indexed database is pushed as fetched, so the sanitization setting + # does not apply to it. + export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[INFO] Started database 2 push." + assert_output_contains "Pushed to ftp with index [2]." + assert_output_contains "[ OK ] Finished database 2 push." + + popd >/dev/null +} + @test "push-db: Skip for a source that does not accept a push" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." stub_sibling "vortex-push-db-s3" "Pushed to s3." export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in url ftp acquia lagoon container_registry none; do + for db_source in url acquia lagoon container_registry none; do export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3." + assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3." assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." done @@ -73,18 +102,35 @@ load ../_helper.bash popd >/dev/null } +@test "push-db: Skip for a dump store source without an upstream source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "[ OK ] Skipped database push as VORTEX_REFRESH_DB_SOURCE is not set, so the ftp source is not a dump store." + assert_output_not_contains "Pushed to ftp." + + popd >/dev/null +} + @test "push-db: Use the source from .env by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." unset VORTEX_FETCH_DB_SOURCE export VORTEX_PUSH_DB_PROCEED=1 run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: gdrive, s3.' - assert_output_not_contains "Pushed to gdrive." + assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3.' + assert_output_not_contains "Pushed to ftp." popd >/dev/null } @@ -92,16 +138,17 @@ load ../_helper.bash @test "push-db: Refuse to push a database that is not sanitized" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-gdrive" "Pushed to gdrive." + stub_sibling "vortex-push-db-ftp" "Pushed to ftp." - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 run .vortex/tooling/src/vortex-push-db assert_failure assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." - assert_output_not_contains "Pushed to gdrive." + assert_output_not_contains "Pushed to ftp." assert_output_not_contains "Finished database push." popd >/dev/null @@ -114,6 +161,7 @@ load ../_helper.bash chmod +x .vortex/tooling/src/vortex-push-db-s3 export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 run .vortex/tooling/src/vortex-push-db diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 785087de4a..9160756ad6 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -236,7 +236,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon, s3 or gdrive." + assert_output_contains "[FAIL] The myorg/db image is built from a database dump file, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to the source of the dump file: url, ftp, acquia, lagoon or s3." assert_output_not_contains "Logged in to the registry." assert_output_not_contains "Fetched the dump file." done @@ -259,7 +259,8 @@ load ../_helper.bash stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -296,7 +297,8 @@ load ../_helper.bash echo "store" >.data/db.sql unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" stub_sibling "vortex-login-container-registry" "Logged in to the registry." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' @@ -316,11 +318,38 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Exports an ftp source database unsanitized without a refresh source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_PUSH_DB_PROCEED=1 + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "[ OK ] Finished database refresh." + + # Without a refresh source, the ftp source is not a dump store. + assert_equal "5" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + @test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="gdrive" + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 @@ -330,7 +359,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to gdrive must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_contains "[FAIL] The database dump file pushed to ftp must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." assert_output_not_contains "Logged in to the registry." assert_equal "0" "$(mock_get_call_num "${mock_docker}")" @@ -345,6 +374,7 @@ load ../_helper.bash unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE export VORTEX_FETCH_DB_SOURCE="s3" + export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 stub_sibling "vortex-login-container-registry" "Logged in to the registry." From 61f8c8a6060504f9da465efdad132b0b05692596 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:37:51 +1100 Subject: [PATCH 12/40] Renamed the S3 push test loop variable that clashed with the BATS status variable. --- .vortex/tooling/tests/unit/push-db-s3.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 72793df610..b29fd798a1 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -603,11 +603,11 @@ setup_s3_push() { mock_set_output "${mock_curl}" $'InternalError\n200' 1 mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 - for status in 200 403; do + for code in 200 403; do run .vortex/tooling/src/vortex-push-db-s3 assert_failure assert_output_contains "Response: " - assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${status}." + assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${code}." assert_output_not_contains "Uploaded the database dump file." done From f97a38722d492f48de925f32434bde50d61b05b1 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:44:00 +1100 Subject: [PATCH 13/40] Wired the FTP and S3 dump store push into CI and the installer, and documented database storage with diagrams. --- .circleci/config.yml | 22 +- .circleci/vortex-test-common.yml | 14 ++ .env | 47 ++-- .env.local.example | 8 - .github/workflows/build-test-deploy.yml | 42 +++- .github/workflows/vortex-test-didi.yml | 84 ++++++- .../content/continuous-integration/README.mdx | 56 +---- .../content/development/database-storage.mdx | 219 ++++++++++++++++++ .../docs/content/development/provision.mdx | 7 +- .../docs/content/development/variables.mdx | 21 +- .vortex/docs/content/features.mdx | 10 +- .vortex/docs/cspell.json | 4 +- .../Prompts/Handlers/DatabaseFetchSource.php | 10 +- .../Prompts/Handlers/MigrationFetchSource.php | 10 +- .../DatabaseFetchSourceHandlerProcessTest.php | 3 - ...MigrationFetchSourceHandlerProcessTest.php | 17 +- 16 files changed, 416 insertions(+), 158 deletions(-) create mode 100644 .vortex/docs/content/development/database-storage.mdx diff --git a/.circleci/config.yml b/.circleci/config.yml index 4647de6b11..7b34398c7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,9 +372,23 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV no_output_timeout: 30m #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -399,10 +413,12 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. VORTEX_PUSH_DB_PROCEED: 1 - #;> DB_PUSH + #;< VORTEX_DEV + # Tell this provider's demo database dumps apart on the test FTP server. + VORTEX_PUSH_DB_FILE_PREFIX: circleci- + #;> VORTEX_DEV # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 #;> !PROVISION_TYPE_PROFILE diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 5d52519f05..71b2bbfeb1 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -192,9 +192,23 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV no_output_timeout: 30m #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + #;> MIGRATION_DB_PUSH + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.env b/.env index eaa76a2e98..41d5604266 100644 --- a/.env +++ b/.env @@ -222,9 +222,9 @@ VORTEX_FETCH_DB_SOURCE=url # VORTEX_DB_IMAGE=your_org/your_site:latest # Source of the database dump file the nightly CI job imports into a clean -# base image to build the database container image: url, ftp, acquia, lagoon, -# s3 or gdrive. The fetch script of that source reads its own variables, such -# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# base image to build the database container image: url, ftp, acquia, lagoon +# or s3. The fetch script of that source reads its own variables, such as +# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -258,21 +258,14 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 -#;< DB_FETCH_SOURCE_GDRIVE -# Database dump file sourced from Google Drive. - -# Google Drive folder ID with the database dump file. -# -# The ID is the last segment of the folder URL. The folder must be in a shared -# drive and shared with the service account. -VORTEX_FETCH_DB_GDRIVE_FOLDER_ID= -#;> DB_FETCH_SOURCE_GDRIVE - #;< DB_PUSH -# Source of the database dump file the nightly CI job sanitizes and pushes to -# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -# Add them, and keep secrets in CI. +# Upstream source of the database dump file, such as url, acquia or lagoon. +# +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the +# fetch source, keeping the dump it replaces as the previous one. The fetch +# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL +# for url. Add them, and keep secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url #;> DB_PUSH @@ -315,9 +308,9 @@ VORTEX_FETCH_DB2_SOURCE=url # Source of the second database dump file the nightly CI job imports into a # clean base image to rebuild the migration database container image: url, -# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its -# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep -# secrets in CI. +# ftp, acquia, lagoon or s3. The fetch script of that source reads its own +# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets +# in CI. # VORTEX_REFRESH_DB2_SOURCE=url #;> MIGRATION_DB_FETCH_SOURCE_CONTAINER_REGISTRY @@ -351,12 +344,14 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 -#;< MIGRATION_DB_FETCH_SOURCE_GDRIVE -# Second database dump file sourced from Google Drive. - -# Google Drive folder ID with the second database dump file. -VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID= -#;> MIGRATION_DB_FETCH_SOURCE_GDRIVE +#;< MIGRATION_DB_PUSH +# Upstream source of the second database dump file, such as url or lagoon. +# +# Uncomment to make the second database fetch source a dump store: the +# nightly CI job fetches the dump from this source and pushes it, as fetched, +# to the fetch source, next to the primary dump. +# VORTEX_REFRESH_DB2_SOURCE=url +#;> MIGRATION_DB_PUSH # Environment to fetch the second database from. # diff --git a/.env.local.example b/.env.local.example index 4f87f00489..b7a2eb4d0c 100644 --- a/.env.local.example +++ b/.env.local.example @@ -69,14 +69,6 @@ VORTEX_FETCH_DB_S3_ACCESS_KEY= VORTEX_FETCH_DB_S3_SECRET_KEY= #;> DB_FETCH_SOURCE_S3 -#;< DB_FETCH_SOURCE_GDRIVE -# Database dump file sourced from Google Drive. -# Service account key: Google Cloud console -> IAM & Admin -> Service accounts -> Keys -> Add key - -# Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. -VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY= -#;> DB_FETCH_SOURCE_GDRIVE - #;< DB_FETCH_SOURCE_CONTAINER_REGISTRY # Database image sourced from a container registry. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index a7eda4a045..4a87d8369d 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -225,13 +225,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. @@ -288,10 +307,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -362,9 +379,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index 17399bfe49..a27fc24f0a 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -64,13 +64,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-fi + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -136,10 +155,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -210,9 +227,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order @@ -608,13 +638,32 @@ jobs: VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: vortex-dev-didi-ii + #;< DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} + VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + #;> DB_FETCH_SOURCE_FTP #;< DB_FETCH_SOURCE_S3 VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} #;> DB_FETCH_SOURCE_S3 - #;< DB_FETCH_SOURCE_GDRIVE - VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY: ${{ secrets.VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY }} - #;> DB_FETCH_SOURCE_GDRIVE + #;< MIGRATION_DB_FETCH_SOURCE_FTP + VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} + VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + #;> MIGRATION_DB_FETCH_SOURCE_FTP + #;< MIGRATION_DB_FETCH_SOURCE_S3 + VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} + VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + #;> MIGRATION_DB_FETCH_SOURCE_S3 + #;< VORTEX_DEV + # FTP server that receives the demo database on the nightly run, under + # a prefix that tells this provider's dumps apart. + VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} + VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} + VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} + VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} + VORTEX_PUSH_DB_FILE_PREFIX: gha- + #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 VORTEX_DB_IMAGE: drevops/vortex-dev-mariadb-drupal-data-demo-destination-11.x @@ -679,10 +728,8 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - #;< DB_PUSH - # Refresh the database dump in the database fetch source. + # Refresh the database dumps in the fetch sources that are dump stores. echo "VORTEX_PUSH_DB_PROCEED=1" - #;> DB_PUSH } >> "$GITHUB_ENV" - name: Create cache keys files for database caching @@ -772,9 +819,22 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db + #;< VORTEX_DEV + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + #;> VORTEX_DEV timeout-minutes: 30 #;> DB_PUSH + #;< MIGRATION_DB_PUSH + # Push the freshly-fetched migration database dump to its fetch source, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + timeout-minutes: 30 + #;> MIGRATION_DB_PUSH + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index dcd492e4f3..dd8be65b5e 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,9 +79,9 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry -- On the nightly run of a project with the `s3` or `gdrive` fetch source, - pushes a sanitized dump to that source (see [Push the sanitized database - to a dump store](#push-the-sanitized-database-to-a-dump-store)) +- On the nightly run of a project with a dump store, pushes the sanitized + dump to the FTP server or S3 bucket (see [Database + storage](../development/database-storage.mdx#fetch-from-a-dump-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly @@ -389,7 +389,7 @@ future **Vortex** update ships. The scheduled run builds a database container image from a dump file. With the `container_registry` fetch source, set `VORTEX_REFRESH_DB_SOURCE` to the -source of that file: `url`, `ftp`, `acquia`, `lagoon`, `s3` or `gdrive`. +source of that file: `url`, `ftp`, `acquia`, `lagoon` or `s3`. `VORTEX_REFRESH_DB_SOURCE` selects a fetch script, and each fetch script reads its own variables. The installer adds to `.env` the variables of the @@ -412,54 +412,6 @@ For the migration database, set `VORTEX_REFRESH_DB2_SOURCE` and the variables apply to both databases. See [Refresh the migration database image](../development/migrations.mdx#refresh-the-migration-database-image). -### Push the sanitized database to a dump store - -A dump store is an `s3` or `gdrive` fetch source that holds a sanitized -copy of your hosting database. Developers and branch builds fetch that -copy and need no hosting access. On the nightly run, the `database` job -fetches a dump from `VORTEX_REFRESH_DB_SOURCE` and sanitizes it. Its -**Push DB** step then uploads the dump to the fetch source. - -To refresh a dump store every night: - -1. In `.env`, set `VORTEX_FETCH_DB_SOURCE` to `s3` or `gdrive`. The - installer sets it and adds the variables of that source. -2. In `.env`, set `VORTEX_REFRESH_DB_SOURCE` to the hosting source: `url`, - `ftp`, `acquia` or `lagoon`. Add the variables of that source from the - table in [Dump file source for a database - image](#dump-file-source-for-a-database-image). -3. Add the credentials of the dump store as CI secrets: - - | Source | Secrets | - |---|---| - | `s3` | `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` | - | `gdrive` | `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY`, with the content of the JSON key file on 1 line, such as the output of `jq -c . key.json` | - - GitHub Actions passes these secrets to the `database` job. Add the - secrets of the hosting source to the `env` block of that job. - -4. After the next nightly run, confirm that its **Push DB** step ends with - `Finished database push.` - -The `s3` push writes to the bucket, region and prefix that the fetch -reads. The `gdrive` push replaces a file with the same name in place, so -the file keeps its ID and sharing. Both pushes stop with an error when -`VORTEX_PROVISION_SANITIZE_DB_SKIP` is `1`. - -The `gdrive` source signs in with a Google Cloud service account key. -Service accounts have no storage of their own, so the folder must be in a -shared drive. To set up the folder: - -1. In the Google Cloud console, enable the Google Drive API and create a - service account with a JSON key. -2. In a shared drive, create a folder and add the service account email - as a **Content manager**. -3. Copy the folder ID, the last segment of the folder URL, into - `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` in `.env`. -4. Give developers the key of a second service account with the - **Viewer** role. They set `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` in - `.env.local` to the path of the key file. - ## Trigger conditions Both providers build on branch pushes, pull requests and tag pushes, plus diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx new file mode 100644 index 0000000000..e0a7dad597 --- /dev/null +++ b/.vortex/docs/content/development/database-storage.mdx @@ -0,0 +1,219 @@ +--- +sidebar_label: Database storage +sidebar_position: 6 +--- + +# Database storage + +Your site's database reaches a developer's machine and CI in 1 of 3 ways. +The fetch source in `VORTEX_FETCH_DB_SOURCE` picks the way. This page +shows how fetching, refreshing and pushing the database work in each case. +It also covers setting up an FTP server or an S3 bucket as a dump store, +so developers get a sanitized dump without access to your hosting. + +The page covers these commands: + +- `ahoy fetch-db` (`vortex-fetch-db`) downloads a dump into `.data` +- `vortex-refresh-db` provisions a site from the fetched dump in CI and + exports it again +- `vortex-push-db` uploads the exported dump to a dump store on the + nightly CI run + +## Fetch from the source + +With the `url`, `ftp`, `acquia`, `lagoon` or `s3` source, every +environment downloads the dump from that source. Developers need +credentials for it. + +```text + Developer CI (every run with no cached dump) + ───────── ────────────────────────────────── + ahoy fetch-db Fetch DB + │ │ + ▼ ▼ + .data/db.sql ◀── source ──▶ .data/db.sql + │ │ + ▼ ▼ + ahoy provision Export DB (vortex-refresh-db) + │ provisions the dump and exports + │ it as fetched + ▼ + cache ──▶ build job +``` + +## Fetch a database image + +With the `container_registry` source, the database lives in a container +image. Developers and CI pull the image. The nightly CI run rebuilds the +image from a dump file and pushes it back. See [Dump file source for a +database +image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). + +```text + Nightly CI + ────────── + Fetch DB ── VORTEX_REFRESH_DB_SOURCE (for example, lagoon) + │ + ▼ + Export DB (vortex-refresh-db) + │ imports the dump into a clean base image + ▼ + Deploy DB image (vortex-push-db-image) ──▶ container registry +``` + +## Fetch from a dump store + +A dump store is an `ftp` or `s3` source with an upstream source set in +`VORTEX_REFRESH_DB_SOURCE`, such as `lagoon`. Developers and branch builds +fetch from the store and need no hosting access. The nightly CI run +refreshes the store from the upstream source: + +```text + Nightly CI (VORTEX_PUSH_DB_PROCEED=1) + ───────────────────────────────────── + Fetch DB + │ fetches from VORTEX_REFRESH_DB_SOURCE in place of the store + ▼ + Export DB (vortex-refresh-db) + │ provisions the dump, sanitizes it and exports it + ▼ + Push DB (vortex-push-db) + │ 1. keeps the current dump as db-previous.sql + │ 2. uploads the new dump as db.sql + ▼ + FTP server or S3 bucket + ├── db.sql ◀── ahoy fetch-db, branch builds + └── db-previous.sql +``` + +The store always holds 2 copies: the current dump and the dump it +replaced. Fetching reads the current dump. To fetch the previous dump from +an FTP store, set `VORTEX_FETCH_DB_FTP_FILE` to its path in `.env.local`, +for example `your_site/db-previous.sql`. + +Every other CI run fetches from the store, so only the nightly run needs +the credentials of the upstream source. + +The pushed dump leaves your hosting and CI, so it's always sanitized. The +nightly run stops with an error when `VORTEX_PROVISION_SANITIZE_DB_SKIP` +is `1`. + +### Multiple databases + +The migration database pushes its own dump file to its own fetch source. +Its files sit next to the primary dump in the same store: + +```text + FTP server or S3 bucket + ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) + ├── db-previous.sql + ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) + └── db2-previous.sql +``` + +To make the migration source a dump store, set +`VORTEX_REFRESH_DB2_SOURCE` to its upstream source. The +**Push migration DB** step pushes the migration dump as fetched, because +**Vortex** sanitizes only the Drupal database. + +## Set up an FTP dump store + +Before you begin, create an FTP account that can write to a directory on +the server. The push uploads the dump under a temporary name, then renames +the files into place. The server must allow the `DELE`, `RNFR` and `RNTO` +commands, which standard FTP servers do. + +1. In `.env`, set the fetch source and the server: + + ```shell + VORTEX_FETCH_DB_SOURCE=ftp + VORTEX_FETCH_DB_FTP_HOST=ftp.example.com + VORTEX_FETCH_DB_FTP_PORT=21 + VORTEX_FETCH_DB_FTP_FILE=your_site/db.sql + ``` + +2. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, for example `lagoon`. Add the variables and secrets of + that source from the table in [Dump file source for a database + image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). +3. Add `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` as CI + secrets. GitHub Actions passes them to the `database` job. CircleCI + reads them from the project environment variables. +4. Ask each developer to set `VORTEX_FETCH_DB_FTP_USER` and + `VORTEX_FETCH_DB_FTP_PASS` in `.env.local`. +5. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the server holds `db.sql`. + +To give CI an account with write access and developers a read-only one, +set the CI account in `VORTEX_PUSH_DB_FTP_USER` and +`VORTEX_PUSH_DB_FTP_PASS`. On GitHub Actions, map those 2 secrets in the +`env` block of the `database` job. + +FTP sends the credentials and the dump unencrypted. Use a server on a +network you trust, or the S3 store. + +## Set up an S3 dump store + +Before you begin, create a private S3 bucket with public access blocked. + +1. Create an IAM user for CI with this policy. Replace `your-bucket` with + your bucket name: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": "arn:aws:s3:::your-bucket/*" + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": "arn:aws:s3:::your-bucket" + } + ] + } + ``` + + The push copies the current dump before the upload. Without + `s3:ListBucket`, S3 answers a missing dump with an access error, and + the first push fails. + +2. In `.env`, set the fetch source and the bucket: + + ```shell + VORTEX_FETCH_DB_SOURCE=s3 + VORTEX_FETCH_DB_S3_BUCKET=your-bucket + VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + VORTEX_FETCH_DB_S3_PREFIX=your_site + ``` + +3. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your + upstream source, and add the variables and secrets of that source. +4. Add the access key of the IAM user as the + `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` CI + secrets. +5. Create a second IAM user for developers with `s3:GetObject` only. Ask + each developer to set its keys in `.env.local`. +6. After the next nightly run, confirm that the **Push DB** step ends with + `Finished database push.` and that the bucket holds `db.sql`. + +S3 copies a dump of up to 5 GB in 1 request, so the push fails for a +larger dump. + +## Tell pushes apart + +`VORTEX_PUSH_DB_FILE_PREFIX` adds a prefix to the names of the pushed +files, such as `gha-db.sql` and `gha-db-previous.sql`. Use a different +prefix in each place that pushes to the same store. A fetch reads the +prefixed files only when its remote filename includes the prefix. + +## What's next + +- [Variables reference](variables.mdx) lists every `VORTEX_PUSH_DB_` + variable +- [Provisioning](provision.mdx) covers importing the fetched dump +- [Continuous integration](../continuous-integration/README.mdx) covers + the `database` job and its caching diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index eecfbc5c06..4117778be9 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -375,14 +375,13 @@ following values: - `container_registry`: a database container image, pulled instead of a file - `s3`: a file in an S3 bucket -- `gdrive`: a file in a Google Drive folder - `none`: no source; you put the dump into `.data` yourself Any other value stops the command with an error that lists these values. -The nightly CI run can refresh an `s3` or `gdrive` source with a sanitized -dump from your hosting. See [Push the sanitized database to a dump -store](../continuous-integration/README.mdx#push-the-sanitized-database-to-a-dump-store). +An `ftp` or `s3` source can also be a dump store that the nightly CI run +refreshes with a sanitized dump. See [Database +storage](database-storage.mdx). Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index d047a26eae..16ba2bf9d5 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -323,7 +323,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_FTP_FILE` | Second database dump FTP file name. | `db2.sql` | `.env` | | `VORTEX_FETCH_DB2_FTP_HOST` | Second database dump FTP host. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_PORT` | Second database dump FTP port. | `21` | `.env` | -| `VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID` | Google Drive folder ID with the second database dump file. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_BUCKET` | AWS S3 bucket name for second database fetch. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env` | | `VORTEX_FETCH_DB2_SOURCE` | Second database fetch source. | `url` | `.env` | @@ -353,10 +352,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_FTP_PASS` | Database dump FTP password. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_PORT` | Database dump FTP port. | `21` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_USER` | Database dump FTP user. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | -| `VORTEX_FETCH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_DB_FILE` | Database dump file name.

The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID with the database dump file.

The ID is the last segment of the folder URL. The folder must be in a shared drive and shared with the service account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-gdrive` | -| `VORTEX_FETCH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-fetch-db-gdrive` | | `VORTEX_FETCH_DB_LAGOON_DB_DIR` | Directory where DB dumps are stored on the host. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_DB_FILE` | Database dump file name on the host. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_LAGOON_PROJECT` | Lagoon project name. | `${LAGOON_PROJECT}` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | @@ -399,7 +394,6 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_FILES_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-files-acquia`, `.vortex/tooling/src/vortex-fetch-files-lagoon` | | `VORTEX_FETCH_FILES_URL` | Public files archive sourced from a URL.

A tar archive, optionally compressed, with the content of the public files directory at its root. HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-files-url` | | `VORTEX_FRONTEND_BUILD_SKIP` | Skip building of the front-end assets inside the container image.

Set to `1` to build theme assets on the host or as part of deployment instead of during the container image build. | `UNDEFINED` | `.env` | -| `VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY` | Path to the Google Cloud service account JSON key file. Keep the file outside the project directory. | `UNDEFINED` | `.env.local.example` | | `VORTEX_IMPORT_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_IMPORT_DB_FILE_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-import-db-file` | | `VORTEX_INSTALLER_INTERACTIVE` | Run installer in interactive mode. | `0` | `.vortex/tooling/src/vortex-update` | @@ -555,10 +549,14 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | -| `VORTEX_PUSH_DB_GDRIVE_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. The same name is used for the file in the Google Drive folder. | `db.sql` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_FOLDER_ID` | Google Drive folder ID to push the database dump file to.

Defaults to the folder the database is fetched from. | `${VORTEX_FETCH_DB_GDRIVE_FOLDER_ID}` | `.vortex/tooling/src/vortex-push-db-gdrive` | -| `VORTEX_PUSH_DB_GDRIVE_SERVICE_ACCOUNT_KEY` | Google Cloud service account key: the JSON content or a path to the JSON key file. | `${VORTEX_GDRIVE_SERVICE_ACCOUNT_KEY}` | `.vortex/tooling/src/vortex-push-db-gdrive` | +| `VORTEX_PUSH_DB_FILE_PREFIX` | Prefix for the names of the pushed database dump files.

Use a different prefix in each place that pushes to the same storage, such as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the prefixed name only when its remote file name includes the prefix. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_HOST` | The FTP host. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PASS` | The FTP password. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_PORT` | The FTP port. | `21` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_USER` | The FTP user. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_PROCEED` | Proceed with push.

Set to `1` in the nightly CI job only, so the storage is refreshed once a day and never from a feature branch build. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_S3_ACCESS_KEY` | AWS access key. | `${S3_ACCESS_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_BUCKET` | S3 bucket name. | `${S3_BUCKET}` | `.vortex/tooling/src/vortex-push-db-s3` | @@ -566,10 +564,11 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_PREFIX` | S3 prefix (path within the bucket). | `${S3_PREFIX}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_REGION` | S3 region. | `${S3_REGION}` | `.vortex/tooling/src/vortex-push-db-s3` | -| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the database dump file name, which is the name the database is fetched with. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the remote file name the database is fetched from, then to the database dump file name. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index 44c2adb358..6880fab076 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -169,8 +169,8 @@ import { - _Lint_ the code without provisioning a site - _Fetch the database_ from the hosting provider and cache it for the - day, with a nightly refresh of the database container image or of a - sanitized dump in S3 or Google Drive + day, with a nightly refresh of the database image or of a sanitized + dump on FTP or S3 - _Build_ and test a fully provisioned site, with [Behat](https://docs.behat.org/en/latest/) tests split across parallel containers @@ -245,8 +245,7 @@ import { ### Other hosting providers - Webhook deployments for any provider that accepts a deployment URL - - Database fetch from a URL, FTP, S3, Google Drive or a container - registry + - Database fetch from a URL, FTP, S3 or a container registry :::note @@ -385,8 +384,7 @@ import { [`drevops/vortex-tooling`](https://packagist.org/packages/drevops/vortex-tooling) Composer package, run the same way in every environment - Customization through environment variables - - Database sourced from the hosting provider, a URL, FTP, S3, Google - Drive or a + - Database sourced from the hosting provider, a URL, FTP, S3 or a container image seeded with [Database data images](https://github.com/drevops/database-data-images) diff --git a/.vortex/docs/cspell.json b/.vortex/docs/cspell.json index 5d4b399675..092cd8aebd 100644 --- a/.vortex/docs/cspell.json +++ b/.vortex/docs/cspell.json @@ -13,12 +13,13 @@ "Buildx", "Diffy", "Dockerfiles", - "GDRIVE", "GHSA", "Incapsula", "MAILHUB", "NRAK", "Olivero", + "RNFR", + "RNTO", "REDIS", "Runsheet", "Runsheets", @@ -54,7 +55,6 @@ "drush", "ergebnis", "fieldsets", - "gdrive", "gherkinlint", "gitleaks", "hadolint", diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index 6fd4967b04..f5abc209ff 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -21,8 +21,6 @@ class DatabaseFetchSource extends AbstractHandler { const S3 = 's3'; - const GDRIVE = 'gdrive'; - const NONE = 'none'; /** @@ -57,7 +55,6 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', - self::GDRIVE => 'Google Drive', self::NONE => 'None', ]; @@ -135,7 +132,6 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, - self::GDRIVE, ]; foreach ($types as $type) { @@ -155,9 +151,9 @@ public function process(): void { File::removeTokenAsync('DB_IN_IMAGE'); } - // DB_PUSH wraps the nightly push of the sanitized dump, which only a - // dump store source accepts. - if (!in_array($v, [self::S3, self::GDRIVE], TRUE)) { + // DB_PUSH wraps the nightly push of the sanitized dump, which only an + // 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { File::removeTokenAsync('DB_PUSH'); } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index abc1fefba6..53c2b31b7c 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -21,8 +21,6 @@ class MigrationFetchSource extends AbstractHandler { const S3 = 's3'; - const GDRIVE = 'gdrive'; - /** * {@inheritdoc} */ @@ -55,7 +53,6 @@ public function options(array $responses): ?array { self::LAGOON => 'Lagoon environment', self::CONTAINER_REGISTRY => 'Container registry', self::S3 => 'S3 bucket', - self::GDRIVE => 'Google Drive', ]; if (isset($responses[HostingProvider::id()])) { @@ -136,7 +133,6 @@ public function process(): void { self::LAGOON, self::CONTAINER_REGISTRY, self::S3, - self::GDRIVE, ]; foreach ($types as $type) { @@ -148,6 +144,12 @@ public function process(): void { File::removeTokenAsync($token); } } + + // MIGRATION_DB_PUSH wraps the nightly push of the migration dump, which + // only an 'ftp' or 's3' source accepts. + if (!in_array($v, [self::FTP, self::S3], TRUE)) { + File::removeTokenAsync('MIGRATION_DB_PUSH'); + } } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php index 0b9cd9d62b..9ca89d1241 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php @@ -36,9 +36,6 @@ public static function dataProviderHandlerProcess(): \Iterator { yield 'db_fetch_source_s3' => [ static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::S3), ]; - yield 'db_fetch_source_gdrive' => [ - static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::GDRIVE), - ]; } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index e410c67712..9c128f4a8a 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -27,6 +27,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-refresh-db'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_IMAGE_BASE'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'Deploy DB image'); + $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_ftp' => [ @@ -39,6 +40,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_acquia' => [ @@ -75,20 +77,7 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST'); - }), - ]; - yield 'migration_fetch_source_gdrive' => [ - static::cw(function (AbstractHandlerProcessTestCase $test): void { - $test->prompts[Migration::id()] = TRUE; - $test->prompts[MigrationFetchSource::id()] = MigrationFetchSource::GDRIVE; - }), - static::cw(function (AbstractHandlerProcessTestCase $test): void { - $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_SOURCE=gdrive'); - $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_GDRIVE_FOLDER_ID='); - $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); - $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); - // The migration database is fetched from the store but never pushed. - $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'vortex-push-db'); + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); }), ]; yield 'migration_fetch_source_container_registry' => [ From 102eeac491740ea63a720f47e7cfd211641b2ff2 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 14:44:07 +1100 Subject: [PATCH 14/40] Updated snapshots. --- .../.github/workflows/build-test-deploy.yml | 2 ++ .../ciprovider_circleci/.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../db_fetch_source_container_registry/.env | 6 ++-- .../.github/workflows/build-test-deploy.yml | 6 ++-- .../handler_process/db_fetch_source_ftp/.env | 21 +++++++++----- .../.github/workflows/build-test-deploy.yml | 25 +++++++++++++++++ .../handler_process/db_fetch_source_s3/.env | 25 +++++++++-------- .../.github/workflows/build-test-deploy.yml | 11 +------- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.env | 6 ++-- .../.github/workflows/build-test-deploy.yml | 8 +++--- .../migration_fetch_source_ftp/.env | 9 +++++- .../.github/workflows/build-test-deploy.yml | 28 +++++++++++++++++-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../migration_fetch_source_s3/.env | 9 +++++- .../.github/workflows/build-test-deploy.yml | 28 +++++++++++++++++-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 10 ++++--- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../timezone_circleci/.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 6 ++-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 6 ++-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 2 ++ .../.github/workflows/build-test-deploy.yml | 10 +++---- 68 files changed, 254 insertions(+), 116 deletions(-) create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index b7059b0f25..3580a1e345 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -241,6 +241,8 @@ jobs: { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - name: Create cache keys files for database caching diff --git a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml index 65a0cc3f96..613d58ed05 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -490,6 +490,17 @@ +@@ -492,6 +492,17 @@ hide_and_recreate: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml index 095f6fc249..ea65c0eaad 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index e6357bf6c3..f98032044d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -14,9 +14,9 @@ +VORTEX_DB_IMAGE=the_empire/star_wars:latest +# Source of the database dump file the nightly CI job imports into a clean -+# base image to build the database container image: url, ftp, acquia, lagoon, -+# s3 or gdrive. The fetch script of that source reads its own variables, such -+# as VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. ++# base image to build the database container image: url, ftp, acquia, lagoon ++# or s3. The fetch script of that source reads its own variables, such as ++# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 5a2acf73b9..3086561937 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +278,15 @@ +@@ -276,6 +280,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index 74fe89268f..a6c57df4f1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,19 @@ +@@ -180,12 +180,27 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,11 +6,8 @@ +VORTEX_FETCH_DB_SOURCE=ftp -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from FTP. - ++ +# Database dump FTP host. +VORTEX_FETCH_DB_FTP_HOST= + @@ -20,10 +17,20 @@ +# Database dump FTP file name. +VORTEX_FETCH_DB_FTP_FILE=db.sql + ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source, keeping the dump it replaces as the previous one. The fetch ++# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL ++# for url. Add them, and keep secrets in CI. ++# VORTEX_REFRESH_DB_SOURCE=url + # Environment to fetch the database from. # - # Applies to hosting environments. -@@ -254,17 +260,3 @@ +@@ -254,17 +269,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..99ebadce06 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -0,0 +1,25 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} ++ VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -276,6 +278,15 @@ + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi + ./vendor/bin/vortex-refresh-db ++ timeout-minutes: 30 ++ ++ # Push the freshly-exported sanitized database dump to the database fetch ++ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the ++ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). ++ - name: Push DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ ./vendor/bin/vortex-push-db + timeout-minutes: 30 + + # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index b26eb7cb6a..63867d4b6e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,22 @@ +@@ -180,12 +180,24 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,27 +6,28 @@ +VORTEX_FETCH_DB_SOURCE=s3 -# Database dump file sourced from a URL. --# --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= +# Database dump file sourced from S3. - ++ +# AWS S3 bucket name for database fetch. +VORTEX_FETCH_DB_S3_BUCKET= + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + -+# Source of the database dump file the nightly CI job sanitizes and pushes to -+# the database fetch source: url, ftp, acquia or lagoon. The fetch script of -+# that source reads its own variables, such as VORTEX_FETCH_DB_URL for url. -+# Add them, and keep secrets in CI. ++# Upstream source of the database dump file, such as url, acquia or lagoon. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Uncomment to make the database fetch source a dump store: the nightly CI ++# job fetches the dump from this source, sanitizes it and pushes it to the ++# fetch source, keeping the dump it replaces as the previous one. The fetch ++# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL ++# for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url -+ + # Environment to fetch the database from. # - # Applies to hosting environments. -@@ -254,17 +263,3 @@ +@@ -254,17 +266,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml index d8ec3cb540..b95ee1f3c8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -7,16 +7,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -241,6 +243,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" -+ # Refresh the database dump in the database fetch source. -+ echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -274,6 +278,15 @@ +@@ -276,6 +278,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml index 70ccf44837..6b2526e489 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index 24407e1e7d..d0b2146bf0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -573,104 +573,3 @@ +@@ -575,104 +575,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml index c4971de6bd..a4f3384e34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -269,6 +269,9 @@ +@@ -271,6 +271,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml index 69a8a20830..657e41c70f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml @@ -312,6 +312,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index fc0db2960c..cdfd843852 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -24,9 +24,9 @@ + +# Source of the second database dump file the nightly CI job imports into a +# clean base image to rebuild the migration database container image: url, -+# ftp, acquia, lagoon, s3 or gdrive. The fetch script of that source reads its -+# own variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep -+# secrets in CI. ++# ftp, acquia, lagoon or s3. The fetch script of that source reads its own ++# variables, such as VORTEX_FETCH_DB2_URL for url. Add them, and keep secrets ++# in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 975da88adb..e819850e04 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" + # Refresh the database dumps in the fetch sources that are dump stores. + echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching -@@ -270,9 +274,13 @@ +@@ -272,9 +276,13 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -23,7 +23,7 @@ ./vendor/bin/vortex-refresh-db timeout-minutes: 30 -@@ -431,6 +439,11 @@ +@@ -433,6 +441,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env index 7f2e72048c..ae7d4bb2fe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,35 @@ +@@ -195,6 +195,42 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -25,6 +25,13 @@ +# Second database dump FTP file name. +VORTEX_FETCH_DB2_FTP_FILE=db2.sql + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml index 4292b4f549..bf29fc7cbc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} ++ VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env index cddcb5610a..b1aec4e0f5 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,32 @@ +@@ -195,6 +195,39 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -22,6 +22,13 @@ +# AWS S3 region. +VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 + ++# Upstream source of the second database dump file, such as url or lagoon. ++# ++# Uncomment to make the second database fetch source a dump store: the ++# nightly CI job fetches the dump from this source and pushes it, as fetched, ++# to the fetch source, next to the primary dump. ++# VORTEX_REFRESH_DB2_SOURCE=url ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml index 4292b4f549..283b4f1f72 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -1,4 +1,13 @@ -@@ -270,6 +270,9 @@ +@@ -191,6 +191,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -272,6 +274,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +17,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -278,6 +283,14 @@ + ./vendor/bin/vortex-refresh-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its fetch source, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -433,6 +446,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml index 4292b4f549..460b6bc35f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -270,6 +270,9 @@ +@@ -272,6 +272,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +434,11 @@ +@@ -433,6 +436,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml index c0b7f62c2c..ab7610ac1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -314,7 +314,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +498,6 @@ +@@ -501,18 +500,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index 042d1c6b55..0dfba85dcf 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -15,7 +15,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -167,130 +164,8 @@ +@@ -167,132 +164,8 @@ run: docker compose exec -T cli bash -c "npm --prefix=\${WEBROOT}/themes/custom/\${DRUPAL_THEME} run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -93,6 +93,8 @@ - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" +- # Refresh the database dumps in the fetch sources that are dump stores. +- echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - - - name: Create cache keys files for database caching @@ -146,7 +148,7 @@ permissions: contents: read -@@ -333,14 +208,6 @@ +@@ -335,14 +208,6 @@ VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING: "1" VORTEX_SSH_REMOVE_ALL_KEYS: "1" VORTEX_DEBUG: ${{ vars.VORTEX_DEBUG }} @@ -161,7 +163,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -379,29 +246,6 @@ +@@ -381,29 +246,6 @@ - name: Install Vortex tooling run: ./scripts/vortex-tooling.sh @@ -191,7 +193,7 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -577,7 +421,6 @@ +@@ -579,7 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml index 962162638f..56e766a652 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml @@ -295,6 +295,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml index 2f4b0324b6..e5104a12f6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -311,14 +307,7 @@ +@@ -313,14 +309,7 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -24,7 +24,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -440,66 +429,6 @@ +@@ -442,66 +431,6 @@ run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} @@ -91,7 +91,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +441,6 @@ +@@ -514,19 +443,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -111,7 +111,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +457,6 @@ +@@ -543,16 +459,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml index 0b83288c75..4125037c62 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml index 452179b6d7..4d1ca474b4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,7 +300,6 @@ +@@ -312,7 +302,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -30,7 +30,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +410,6 @@ +@@ -423,7 +412,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -38,7 +38,7 @@ - name: Provision site run: | -@@ -434,11 +422,6 @@ +@@ -436,11 +424,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml index 0d5dd2cae9..d21bef58a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml @@ -286,6 +286,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml index 8ffe668f17..3245313f2f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,9 +300,7 @@ +@@ -312,9 +302,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -32,7 +32,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -421,7 +409,6 @@ +@@ -423,7 +411,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -40,7 +40,7 @@ - name: Provision site run: | -@@ -435,11 +422,6 @@ +@@ -437,11 +424,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -52,7 +52,7 @@ - name: Test with PHPUnit if: ${{ env.VORTEX_CI_IS_PHPUNIT_RUNNER == 'true' }} run: docker compose exec -T cli vendor/bin/phpunit -@@ -499,18 +481,6 @@ +@@ -501,18 +483,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml index 87d8397465..8e015f51d7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml @@ -277,6 +277,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml index 0b32b5b70d..fd50aad45b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -313,12 +309,6 @@ +@@ -315,12 +311,6 @@ VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -22,7 +22,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -511,19 +501,6 @@ +@@ -513,19 +503,6 @@ exit 1 fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml index 194ac3218b..4735766d37 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml index 3cf240711d..54dc4477e3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml @@ -298,6 +298,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml index f4e7a01167..03afc8216b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Build stack run: docker compose up --no-deps --detach cli -@@ -270,6 +266,9 @@ +@@ -272,6 +268,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -19,7 +19,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -431,6 +430,11 @@ +@@ -433,6 +432,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml index 64bcd3e9ab..fffe322f5d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml @@ -291,6 +291,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml index 37281e8ff5..121b9f5477 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml @@ -300,6 +300,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml index a4847a4654..2963f464ca 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -310,7 +310,6 @@ +@@ -312,7 +312,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -421,7 +420,6 @@ +@@ -423,7 +422,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -14,7 +14,7 @@ - name: Provision site run: | -@@ -434,11 +432,6 @@ +@@ -436,11 +434,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml index 52fdedfed1..8cbd50d31b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml index f2aacd0e81..3262246832 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml index bf65b2a0ba..8fc685f729 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml index 83fae1f090..4bf951f78a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -311,7 +311,6 @@ +@@ -313,7 +313,6 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true -@@ -439,66 +438,6 @@ +@@ -441,66 +440,6 @@ if: ${{ env.VORTEX_CI_IS_JEST_RUNNER == 'true' }} run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml index affec2423a..05b7fbe608 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml index 65af74315e..dfdbda3a2a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml index f56bdf626c..b59dde79d2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml @@ -307,6 +307,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml index da2e5188f3..0cd1802313 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +307,6 @@ +@@ -314,7 +309,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -499,18 +493,6 @@ +@@ -501,18 +495,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml index 5001ef54a4..f2890ba107 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml @@ -303,6 +303,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml index ecbb0ad695..e3c358bb75 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml @@ -58,7 +58,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -310,15 +269,7 @@ +@@ -312,15 +271,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -74,7 +74,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -421,7 +372,6 @@ +@@ -423,7 +374,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -82,7 +82,7 @@ - name: Provision site run: | -@@ -435,71 +385,6 @@ +@@ -437,71 +387,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -154,7 +154,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -512,19 +397,6 @@ +@@ -514,19 +399,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -174,7 +174,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -541,16 +413,6 @@ +@@ -543,16 +415,6 @@ path: .logs include-hidden-files: true if-no-files-found: error From fc70a278a09db5ae84e4729e9d60494de2f03d28 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:14:45 +1100 Subject: [PATCH 15/40] Pushed the latest dump under the remote file name with an optional suffix and an opt-in timestamped copy, in place of the previous copy. --- .circleci/config.yml | 6 +- .env | 10 +- .github/workflows/build-test-deploy.yml | 6 +- .../content/development/database-storage.mdx | 51 ++++----- .vortex/tooling/playground/try-push-db-ftp.sh | 4 +- .vortex/tooling/src/vortex-push-db-ftp | 88 +++++++++------ .vortex/tooling/src/vortex-push-db-s3 | 97 +++++++++-------- .vortex/tooling/tests/_helper.bash | 3 +- .vortex/tooling/tests/unit/push-db-ftp.bats | 102 ++++++++++++++---- .vortex/tooling/tests/unit/push-db-s3.bats | 84 +++++++-------- 10 files changed, 267 insertions(+), 184 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b34398c7e..7985a20971 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -416,8 +416,10 @@ jobs: # Refresh the database dumps in the fetch sources that are dump stores. VORTEX_PUSH_DB_PROCEED: 1 #;< VORTEX_DEV - # Tell this provider's demo database dumps apart on the test FTP server. - VORTEX_PUSH_DB_FILE_PREFIX: circleci- + # Tell this provider's demo database dumps apart on the test FTP server, + # with a timestamped copy of each push. + VORTEX_PUSH_DB_FILE_SUFFIX: _circleci + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 diff --git a/.env b/.env index 41d5604266..661a9a946b 100644 --- a/.env +++ b/.env @@ -263,10 +263,14 @@ VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 # # Uncomment to make the database fetch source a dump store: the nightly CI # job fetches the dump from this source, sanitizes it and pushes it to the -# fetch source, keeping the dump it replaces as the previous one. The fetch -# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -# for url. Add them, and keep secrets in CI. +# fetch source as the latest dump. The fetch script of this source reads its +# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep +# secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url + +# Also push a copy of each dump named with a UTC timestamp, such as +# db_20261111-180000.sql, to keep a history of the pushed dumps. +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 #;> DB_PUSH # Environment to fetch the database from. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 4a87d8369d..d126090f7d 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -243,13 +243,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV steps: diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index e0a7dad597..0780d95a3f 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -78,18 +78,21 @@ refreshes the store from the upstream source: │ provisions the dump, sanitizes it and exports it ▼ Push DB (vortex-push-db) - │ 1. keeps the current dump as db-previous.sql - │ 2. uploads the new dump as db.sql + │ 1. replaces db.sql with the new dump + │ 2. with VORTEX_PUSH_DB_TIMESTAMPED_COPY=1, adds a timestamped copy ▼ FTP server or S3 bucket - ├── db.sql ◀── ahoy fetch-db, branch builds - └── db-previous.sql + ├── db.sql ◀── ahoy fetch-db, branch builds + ├── db_20261110-180000.sql timestamped copies, opt-in + └── db_20261111-180000.sql ``` -The store always holds 2 copies: the current dump and the dump it -replaced. Fetching reads the current dump. To fetch the previous dump from -an FTP store, set `VORTEX_FETCH_DB_FTP_FILE` to its path in `.env.local`, -for example `your_site/db-previous.sql`. +`db.sql` always holds the latest dump, and fetching downloads it. By +default, it's the only file a push creates. To keep a history of the +pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push +then also creates a copy named with its UTC time, identical to `db.sql` +at that moment. Nothing removes the copies, so clear out the old ones +yourself. Every other CI run fetches from the store, so only the nightly run needs the credentials of the upstream source. @@ -105,10 +108,10 @@ Its files sit next to the primary dump in the same store: ```text FTP server or S3 bucket - ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) - ├── db-previous.sql - ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) - └── db2-previous.sql + ├── db.sql primary database (VORTEX_FETCH_DB_FTP_FILE) + ├── db_20261111-180000.sql + ├── db2.sql migration database (VORTEX_FETCH_DB2_FTP_FILE) + └── db2_20261111-180000.sql ``` To make the migration source a dump store, set @@ -120,7 +123,7 @@ To make the migration source a dump store, set Before you begin, create an FTP account that can write to a directory on the server. The push uploads the dump under a temporary name, then renames -the files into place. The server must allow the `DELE`, `RNFR` and `RNTO` +it into place, so a failed upload keeps the latest dump. The server must allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers do. 1. In `.env`, set the fetch source and the server: @@ -167,20 +170,11 @@ Before you begin, create a private S3 bucket with public access blocked. "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject"], "Resource": "arn:aws:s3:::your-bucket/*" - }, - { - "Effect": "Allow", - "Action": ["s3:ListBucket"], - "Resource": "arn:aws:s3:::your-bucket" } ] } ``` - The push copies the current dump before the upload. Without - `s3:ListBucket`, S3 answers a missing dump with an access error, and - the first push fails. - 2. In `.env`, set the fetch source and the bucket: ```shell @@ -200,15 +194,16 @@ Before you begin, create a private S3 bucket with public access blocked. 6. After the next nightly run, confirm that the **Push DB** step ends with `Finished database push.` and that the bucket holds `db.sql`. -S3 copies a dump of up to 5 GB in 1 request, so the push fails for a -larger dump. +S3 makes the timestamped copy on its side, so the dump is uploaded once. +S3 copies a dump of up to 5 GB in 1 request, so the timestamped copy +fails for a larger dump. On FTP, the timestamped copy is a second upload. ## Tell pushes apart -`VORTEX_PUSH_DB_FILE_PREFIX` adds a prefix to the names of the pushed -files, such as `gha-db.sql` and `gha-db-previous.sql`. Use a different -prefix in each place that pushes to the same store. A fetch reads the -prefixed files only when its remote filename includes the prefix. +`VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the +pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a +different suffix in each place that pushes to the same store. A fetch reads +the suffixed file only when its remote filename includes the suffix. ## What's next diff --git a/.vortex/tooling/playground/try-push-db-ftp.sh b/.vortex/tooling/playground/try-push-db-ftp.sh index dda38af769..1716701ae0 100755 --- a/.vortex/tooling/playground/try-push-db-ftp.sh +++ b/.vortex/tooling/playground/try-push-db-ftp.sh @@ -3,8 +3,8 @@ # Manual test script for FTP database push. # # Uploads a database dump file to an FTP server using push-db-ftp. Run it -# twice to check that the second run keeps the first dump as -# 'db-previous.sql' next to the new 'db.sql'. +# twice to check that the second run replaces 'db.sql'. Set +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 to also upload a timestamped copy. # # Usage: # export FTP_HOST="ftp.example.com" diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp index 2979a1e1ab..f2abb91e23 100755 --- a/.vortex/tooling/src/vortex-push-db-ftp +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -2,13 +2,14 @@ ## # Push DB dump to FTP. # -# Keeps 2 copies of the database dump file on the FTP server: the current -# dump under the remote file name and the dump it replaced with a '-previous' -# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the -# current dump under the remote file name. +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the dump is also uploaded under a +# name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. # -# The dump is uploaded under a temporary name and renamed into place only -# after the upload succeeds, so a failed upload keeps the current dump. +# Each file is uploaded under a temporary name and renamed into place only +# after the upload succeeds, so a failed upload keeps the existing file. # # IMPORTANT! This script runs outside the container on the host system. # @@ -64,12 +65,18 @@ _vs="VORTEX_FETCH_DB${_db_index}_FILE" _vss="VORTEX_DB${_db_index}_FILE" VORTEX_PUSH_DB_FTP_DB_FILE="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_FILE:-db.sql}}}}" -# Prefix for the names of the pushed database dump files. +# Suffix for the names of the pushed database dump files. # -# Use a different prefix in each place that pushes to the same storage, such -# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the -# prefixed name only when its remote file name includes the prefix. -VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. +# +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" # ------------------------------------------------------------------------------ @@ -87,6 +94,22 @@ curl_config_quote() { printf '"%s"' "${value//\"/\\\"}" } +# Upload the database dump file to a name in the remote directory. +# +# The credentials go to curl on stdin, so they never appear in the process +# list. The commands after the upload run in the remote directory: the +# existing file is removed and the upload takes its name. +ftp_upload() { + local name="${1}" + local upload_name="${name}.uploading" + + echo "user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ + --quote "-*DELE ${name}" \ + --quote "-RNFR ${upload_name}" \ + --quote "-RNTO ${name}" \ + "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" +} + # shellcheck disable=SC2043 for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done @@ -105,36 +128,31 @@ if [[ ${remote_name} == */* ]]; then remote_name="${remote_name##*/}" fi -current_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +base="${remote_name}" +ext="" if [[ ${remote_name} == *.* ]]; then - previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" -else - previous_name="${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" + base="${remote_name%.*}" + ext=".${remote_name##*.}" fi -upload_name="${current_name}.uploading" +current_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_name="${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" info "Started database${_db_index:+ ${_db_index}} dump push to FTP." -note "Local file: ${local_file}" -note "Remote file: ${remote_dir}${current_name}" -note "Previous file: ${remote_dir}${previous_name}" -note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" +note "Local file: ${local_file}" +note "Remote file: ${remote_dir}${current_name}" +note "FTP host: ${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}" -# The credentials go to curl on stdin, so they never appear in the process -# list. The commands after the upload run in the remote directory: the -# previous dump is replaced by the current one, and the upload takes the -# current name. Only the final rename must succeed, since the first push has -# no current or previous dump. -task "Uploading the database dump file and keeping the dump it replaces." -curl_config="user = $(curl_config_quote "${VORTEX_PUSH_DB_FTP_USER}:${VORTEX_PUSH_DB_FTP_PASS}")" -echo "${curl_config}" | curl --silent --show-error --config - --ftp-create-dirs --upload-file "${local_file}" \ - --quote "-*DELE ${previous_name}" \ - --quote "-*RNFR ${current_name}" \ - --quote "-*RNTO ${previous_name}" \ - --quote "-RNFR ${upload_name}" \ - --quote "-RNTO ${current_name}" \ - "ftp://${VORTEX_PUSH_DB_FTP_HOST}:${VORTEX_PUSH_DB_FTP_PORT}/${remote_dir}${upload_name}" || fail "Unable to push the database dump file to FTP." -pass "Uploaded the database dump file and kept the dump it replaced." +task "Uploading the database dump file." +ftp_upload "${current_name}" || fail "Unable to push the database dump file to FTP." +pass "Uploaded the database dump file." + +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # FTP has no copy command, so the copy is a second upload. + task "Uploading the timestamped copy ${remote_dir}${timestamped_name}." + ftp_upload "${timestamped_name}" || fail "Unable to push the timestamped copy of the database dump file to FTP." + pass "Uploaded the timestamped copy ${remote_dir}${timestamped_name}." +fi # Enable xtrace only after the request so the credentials are never written # to the trace output. diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 4463c14f4f..c58857fa4a 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -4,13 +4,11 @@ # # Uses AWS Signature Version 4 with curl (no AWS CLI required). # -# Keeps 2 copies of the database dump file in the bucket: the current dump -# under the remote file name and the dump it replaced with a '-previous' -# suffix, such as 'db.sql' and 'db-previous.sql'. The fetch always finds the -# current dump under the remote file name. -# -# The current dump is copied to the previous one on the S3 side before the -# upload, so a failed upload keeps the current dump. +# Uploads the database dump file under the remote file name, such as +# 'db.sql', which always holds the latest dump that the fetch downloads. With +# VORTEX_PUSH_DB_TIMESTAMPED_COPY set to 1, the uploaded object is also copied +# to a name with a UTC timestamp, such as 'db_20261111-180000.sql', to keep a +# history. # # IMPORTANT! This script runs outside the container on the host system. # @@ -77,12 +75,18 @@ VORTEX_PUSH_DB_S3_REMOTE_FILE="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_DB_FILE}}}" # S3 storage class. VORTEX_PUSH_DB_S3_STORAGE_CLASS="${VORTEX_PUSH_DB_S3_STORAGE_CLASS:-STANDARD}" -# Prefix for the names of the pushed database dump files. +# Suffix for the names of the pushed database dump files. +# +# Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. +# Use a different suffix in each place that pushes to the same storage. A +# fetch reads the suffixed name only when its remote file name includes it. +VORTEX_PUSH_DB_FILE_SUFFIX="${VORTEX_PUSH_DB_FILE_SUFFIX:-}" + +# Also push a copy of the database dump file named with a UTC timestamp. # -# Use a different prefix in each place that pushes to the same storage, such -# as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the -# prefixed name only when its remote file name includes the prefix. -VORTEX_PUSH_DB_FILE_PREFIX="${VORTEX_PUSH_DB_FILE_PREFIX:-}" +# Set to 1 to keep a history of the pushed dumps. The fetch never reads the +# timestamped copies. +VORTEX_PUSH_DB_TIMESTAMPED_COPY="${VORTEX_PUSH_DB_TIMESTAMPED_COPY:-}" # ------------------------------------------------------------------------------ @@ -198,8 +202,8 @@ info "Started database${_db_index:+ ${_db_index}} dump push to S3." [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && VORTEX_PUSH_DB_S3_PREFIX="${VORTEX_PUSH_DB_S3_PREFIX%/}/" -# The file name prefix and the '-previous' suffix apply to the file name, not -# to the directories in the remote file name. +# The suffix and the timestamp apply to the file name, not to the directories +# in the remote file name. remote_dir="" remote_name="${VORTEX_PUSH_DB_S3_REMOTE_FILE}" if [[ ${remote_name} == */* ]]; then @@ -207,46 +211,25 @@ if [[ ${remote_name} == */* ]]; then remote_name="${remote_name##*/}" fi -current_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}" +base="${remote_name}" +ext="" if [[ ${remote_name} == *.* ]]; then - previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name%.*}-previous.${remote_name##*.}" -else - previous_file="${remote_dir}${VORTEX_PUSH_DB_FILE_PREFIX}${remote_name}-previous" + base="${remote_name%.*}" + ext=".${remote_name##*.}" fi +current_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}${ext}" +timestamped_file="${remote_dir}${base}${VORTEX_PUSH_DB_FILE_SUFFIX}_$(date -u +%Y%m%d-%H%M%S)${ext}" object_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${current_file}")" -previous_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}")" +timestamped_key="$(uri_encode_key "${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}")" note "Local file: ${local_file}" note "Remote file: ${VORTEX_PUSH_DB_S3_PREFIX}${current_file}" -note "Previous file: ${VORTEX_PUSH_DB_S3_PREFIX}${previous_file}" note "S3 bucket: ${VORTEX_PUSH_DB_S3_BUCKET}" note "S3 region: ${VORTEX_PUSH_DB_S3_REGION}" [ -n "${VORTEX_PUSH_DB_S3_PREFIX}" ] && note "S3 prefix: ${VORTEX_PUSH_DB_S3_PREFIX}" note "Storage class: ${VORTEX_PUSH_DB_S3_STORAGE_CLASS}" -# A copy can fail after S3 has sent a 200 status, so a success needs the copy -# result in the body. A missing current dump means this is the first push. -task "Keeping the current database dump as the previous one." -set +e -response=$(s3_put "${previous_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") -curl_status=$? -set -e - -[ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." - -http_code=$(echo "${response}" | tail -1) -response_body=$(echo "${response}" | sed '$d') - -if [[ ${http_code} == 2* ]] && [[ ${response_body} == *"/dev/null; then content_type="$(file --brief --mime-type "${local_file}")" else @@ -261,10 +244,6 @@ response=$(s3_put "${object_key}" "${payload_hash}" "${local_file}" "content-typ curl_status=$? set -e -# Enable xtrace only after the signed requests so credentials and the -# signatures are never written to the trace output. -[ "${VORTEX_DEBUG-}" = "1" ] && set -x - [ "${curl_status}" -ne 0 ] && fail "S3 push request failed with curl exit ${curl_status}." http_code=$(echo "${response}" | tail -1) @@ -279,4 +258,30 @@ case "${http_code}" in esac pass "Uploaded the database dump file." +if [ "${VORTEX_PUSH_DB_TIMESTAMPED_COPY}" = "1" ]; then + # The copy runs on the S3 side, so the dump is uploaded only once. A copy + # can fail after S3 has sent a 200 status, so a success needs the copy + # result in the body. + task "Copying the database dump file to ${VORTEX_PUSH_DB_S3_PREFIX}${timestamped_file}." + set +e + response=$(s3_put "${timestamped_key}" "$(hash_sha256 "")" "" "x-amz-copy-source:/${VORTEX_PUSH_DB_S3_BUCKET}/${object_key}") + curl_status=$? + set -e + + [ "${curl_status}" -ne 0 ] && fail "S3 copy request failed with curl exit ${curl_status}." + + http_code=$(echo "${response}" | tail -1) + response_body=$(echo "${response}" | sed '$d') + + if [[ ${http_code} != 2* ]] || [[ ${response_body} != *"/dev/null || exit 1 mkdir -p .data @@ -30,11 +30,11 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_success assert_output_contains "[INFO] Started database dump push to FTP." - assert_output_contains "Local file: ./.data/db.sql" - assert_output_contains "Remote file: db.sql" - assert_output_contains "Previous file: db-previous.sql" - assert_output_contains "FTP host: ftp.example.com:21" - assert_output_contains "[ OK ] Uploaded the database dump file and kept the dump it replaced." + assert_output_contains "Local file: ./.data/db.sql" + assert_output_contains "Remote file: db.sql" + assert_output_contains "FTP host: ftp.example.com:21" + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "timestamped copy" assert_output_contains "[ OK ] Finished database dump push to FTP." assert_output_not_contains "fetch-pass" @@ -42,8 +42,9 @@ setup_ftp_fetch_values() { args="$(mock_get_call_args "${mock_curl}" 1)" assert_string_contains "${args}" "--config - --ftp-create-dirs --upload-file ./.data/db.sql" - # The upload takes a temporary name, and the rename into place comes last. - assert_string_contains "${args}" "--quote -*DELE db-previous.sql --quote -*RNFR db.sql --quote -*RNTO db-previous.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" + # The upload takes a temporary name and replaces the latest dump only after + # it succeeds. + assert_string_contains "${args}" "--quote -*DELE db.sql --quote -RNFR db.sql.uploading --quote -RNTO db.sql" assert_string_contains "${args}" "ftp://ftp.example.com:21/db.sql.uploading" # The credentials reach curl on stdin, never as arguments. assert_string_not_contains "${args}" "fetch-pass" @@ -54,7 +55,38 @@ setup_ftp_fetch_values() { popd >/dev/null } -@test "push-db-ftp: Push to a remote directory with a file name prefix" { +@test "push-db-ftp: Upload a timestamped copy when enabled" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Uploaded the timestamped copy db_" + assert_output_contains "[ OK ] Finished database dump push to FTP." + + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + + # The latest dump is uploaded first, then the copy under its own name. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp.example.com:21/db.sql.uploading" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--upload-file ./.data/db.sql" + [[ ${copy_args} =~ --quote\ -RNTO\ db_[0-9]{8}-[0-9]{6}\.sql ]] + [[ ${copy_args} =~ ftp://ftp\.example\.com:21/db_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to a remote directory with a file name suffix" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data @@ -64,20 +96,22 @@ setup_ftp_fetch_values() { export VORTEX_PUSH_DB_FTP_HOST="push.example.com" export VORTEX_PUSH_DB_FTP_PORT="2121" export VORTEX_PUSH_DB_FTP_FILE="/backups/site/db.sql" - export VORTEX_PUSH_DB_FILE_PREFIX="gha-" + export VORTEX_PUSH_DB_FILE_SUFFIX="_gha" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") run .vortex/tooling/src/vortex-push-db-ftp assert_success - assert_output_contains "Remote file: backups/site/gha-db.sql" - assert_output_contains "Previous file: backups/site/gha-db-previous.sql" - assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "Remote file: backups/site/db_gha.sql" + assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "[ OK ] Uploaded the timestamped copy backups/site/db_gha_" - args="$(mock_get_call_args "${mock_curl}" 1)" # The rename commands run in the remote directory, so they use bare names. - assert_string_contains "${args}" "--quote -*DELE gha-db-previous.sql --quote -*RNFR gha-db.sql --quote -*RNTO gha-db-previous.sql --quote -RNFR gha-db.sql.uploading --quote -RNTO gha-db.sql" - assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/gha-db.sql.uploading" + args="$(mock_get_call_args "${mock_curl}" 1)" + assert_string_contains "${args}" "--quote -*DELE db_gha.sql --quote -RNFR db_gha.sql.uploading --quote -RNTO db_gha.sql" + assert_string_contains "${args}" "ftp://push.example.com:2121/backups/site/db_gha.sql.uploading" + [[ $(mock_get_call_args "${mock_curl}" 2) =~ ftp://push\.example\.com:2121/backups/site/db_gha_[0-9]{8}-[0-9]{6}\.sql\.uploading ]] rm -rf .data @@ -105,9 +139,8 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_success assert_output_contains "[INFO] Started database 2 dump push to FTP." - assert_output_contains "Local file: ./.data/db2.sql" - assert_output_contains "Remote file: db2.sql" - assert_output_contains "Previous file: db2-previous.sql" + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: db2.sql" assert_output_contains "[ OK ] Finished database 2 dump push to FTP." assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp2.example.com:21/db2.sql.uploading" @@ -182,13 +215,14 @@ setup_ftp_fetch_values() { popd >/dev/null } -@test "push-db-ftp: Fail when the upload or a rename fails" { +@test "push-db-ftp: Fail when the upload fails" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "database content" >.data/db.sql setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") mock_set_status "${mock_curl}" 21 1 @@ -196,7 +230,31 @@ setup_ftp_fetch_values() { run .vortex/tooling/src/vortex-push-db-ftp assert_failure assert_output_contains "[FAIL] Unable to push the database dump file to FTP." - assert_output_not_contains "Finished database dump push to FTP." + assert_output_not_contains "Uploaded the timestamped copy" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Fail when the timestamped copy upload fails" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + + mock_curl=$(mock_command "curl") + mock_set_status "${mock_curl}" 0 1 + mock_set_status "${mock_curl}" 25 2 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[FAIL] Unable to push the timestamped copy of the database dump file to FTP." rm -rf .data diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index b29fd798a1..f9c066e57f 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -21,7 +21,6 @@ load ../_helper.bash "S3 region: ap-southeast-2" "Storage class: STANDARD" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -62,7 +61,6 @@ load ../_helper.bash "Local file: ./.data/db.sql" "Remote file: db.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -102,7 +100,6 @@ load ../_helper.bash "S3 bucket: shortcut-bucket" "S3 region: us-east-1" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -146,7 +143,6 @@ load ../_helper.bash "Remote file: backup/db_latest.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -187,7 +183,6 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -228,7 +223,6 @@ load ../_helper.bash "Remote file: backups/daily/db.sql" "S3 prefix: backups/daily/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -267,7 +261,6 @@ load ../_helper.bash declare -a STEPS=( "[INFO] Started database dump push to S3." - "@curl * # 0 # \n200" "@curl * # 0 # AccessDenied\n403" "[FAIL] Unable to push to S3: HTTP status 403." @@ -446,7 +439,6 @@ load ../_helper.bash "S3 region: eu-west-1" "S3 prefix: store/" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -492,7 +484,6 @@ load ../_helper.bash "Local file: ./.data/db_fetched.sql" "Remote file: db_fetched.sql" - "@curl * # 0 # \n200" "@curl * # 0 # 200" "[ OK ] Finished database dump push to S3." @@ -535,36 +526,27 @@ setup_s3_push() { export VORTEX_PUSH_DB_S3_PREFIX="backups" export VORTEX_PUSH_DB_S3_DB_DIR=".data" export VORTEX_PUSH_DB_S3_DB_FILE="db.sql" - unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_PREFIX + unset VORTEX_PUSH_DB_S3_REMOTE_FILE VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY } -@test "push-db-s3: Keep the current dump as the previous one before the upload" { +@test "push-db-s3: Push only the latest dump by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'\n200' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "Remote file: backups/db.sql" - assert_output_contains "Previous file: backups/db-previous.sql" - assert_output_contains "[ OK ] Kept the current database dump as the previous one." assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_not_contains "Copying the database dump file" assert_output_contains "[ OK ] Finished database dump push to S3." - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + assert_equal "1" "$(mock_get_call_num "${mock_curl}")" - # The copy names the current dump as its source and sends no body. - copy_args="$(mock_get_call_args "${mock_curl}" 1)" - assert_string_contains "${copy_args}" "--request PUT" - assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" - assert_string_not_contains "${copy_args}" "--upload-file" - assert_string_contains "${copy_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db-previous.sql" - - upload_args="$(mock_get_call_args "${mock_curl}" 2)" + upload_args="$(mock_get_call_args "${mock_curl}" 1)" assert_string_contains "${upload_args}" "--upload-file .data/db.sql" assert_string_contains "${upload_args}" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" @@ -573,52 +555,67 @@ setup_s3_push() { popd >/dev/null } -@test "push-db-s3: Upload the first dump when there is no current one" { +@test "push-db-s3: Copy the upload to a timestamped copy when enabled" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'NoSuchKey\n404' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 run .vortex/tooling/src/vortex-push-db-s3 assert_success - assert_output_contains "[ OK ] Skipped keeping the previous database dump as there is no current one." + assert_output_contains "[ OK ] Uploaded the database dump file." + assert_output_contains "[ OK ] Copied the database dump file to backups/db_" assert_output_contains "[ OK ] Finished database dump push to S3." + assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + # The copy runs after the upload, names the latest dump as its source and + # sends no body. + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://test-bucket.s3.ap-southeast-2.amazonaws.com/backups/db.sql" + copy_args="$(mock_get_call_args "${mock_curl}" 2)" + assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" + assert_string_not_contains "${copy_args}" "--upload-file" + [[ ${copy_args} =~ https://test-bucket\.s3\.ap-southeast-2\.amazonaws\.com/backups/db_[0-9]{8}-[0-9]{6}\.sql ]] + rm -rf .data popd >/dev/null } -@test "push-db-s3: Fail without uploading when the current dump cannot be kept" { +@test "push-db-s3: Fail when the timestamped copy cannot be made" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 setup_s3_push + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 - # A copy error can come with a 200 status. Each run makes 1 request. + # A copy error can come with a 200 status. Each run makes 2 requests. mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'InternalError\n200' 1 - mock_set_output "${mock_curl}" $'AccessDenied\n403' 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'InternalError\n200' 2 + mock_set_output "${mock_curl}" "200" 3 + mock_set_output "${mock_curl}" $'AccessDenied\n403' 4 for code in 200 403; do run .vortex/tooling/src/vortex-push-db-s3 assert_failure + assert_output_contains "[ OK ] Uploaded the database dump file." assert_output_contains "Response: " - assert_output_contains "[FAIL] Unable to keep the current database dump as the previous one: HTTP status ${code}." - assert_output_not_contains "Uploaded the database dump file." + assert_output_contains "[FAIL] Unable to copy the database dump file to the timestamped copy: HTTP status ${code}." + assert_output_not_contains "Finished database dump push to S3." done - assert_equal "2" "$(mock_get_call_num "${mock_curl}")" + assert_equal "4" "$(mock_get_call_num "${mock_curl}")" rm -rf .data popd >/dev/null } -@test "push-db-s3: Push an indexed database with a file name prefix" { +@test "push-db-s3: Push an indexed database with a file name suffix" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data @@ -632,23 +629,24 @@ setup_s3_push() { export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" export VORTEX_FETCH_DB2_S3_PREFIX="" export VORTEX_FETCH_DB2_FILE="db2.sql" - export VORTEX_PUSH_DB_FILE_PREFIX="circleci-" + export VORTEX_PUSH_DB_FILE_SUFFIX="_circleci" + export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 mock_curl=$(mock_command "curl") - mock_set_output "${mock_curl}" $'\n200' 1 - mock_set_output "${mock_curl}" "200" 2 + mock_set_output "${mock_curl}" "200" 1 + mock_set_output "${mock_curl}" $'\n200' 2 run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "[INFO] Started database 2 dump push to S3." assert_output_contains "Local file: ./.data/db2.sql" - assert_output_contains "Remote file: circleci-db2.sql" - assert_output_contains "Previous file: circleci-db2-previous.sql" + assert_output_contains "Remote file: db2_circleci.sql" assert_output_contains "S3 bucket: db2-bucket" + assert_output_contains "[ OK ] Copied the database dump file to db2_circleci_" assert_output_contains "[ OK ] Finished database 2 dump push to S3." - assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "--header x-amz-copy-source: /db2-bucket/circleci-db2.sql" - assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/circleci-db2.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://db2-bucket.s3.eu-west-1.amazonaws.com/db2_circleci.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "--header x-amz-copy-source: /db2-bucket/db2_circleci.sql" rm -rf .data From 2ea03ee25aa407c3a0f4435841b0a7ffc777a701 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:17:50 +1100 Subject: [PATCH 16/40] Updated the database storage docs and the generated CI configs for the timestamped copy. --- .github/workflows/vortex-test-didi.yml | 12 ++++++++---- .../docs/content/development/database-storage.mdx | 5 +++-- .vortex/docs/content/development/variables.mdx | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index a27fc24f0a..b2b4d23cb4 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -82,13 +82,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url VORTEX_FETCH_DB_FORCE: 1 @@ -656,13 +658,15 @@ jobs: #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV # FTP server that receives the demo database on the nightly run, under - # a prefix that tells this provider's dumps apart. + # a suffix that tells this provider's dumps apart, with a timestamped + # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_PREFIX: gha- + VORTEX_PUSH_DB_FILE_SUFFIX: _gha + VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry VORTEX_FETCH_DB_FORCE: 1 diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 0780d95a3f..b4f9044238 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -123,8 +123,9 @@ To make the migration source a dump store, set Before you begin, create an FTP account that can write to a directory on the server. The push uploads the dump under a temporary name, then renames -it into place, so a failed upload keeps the latest dump. The server must allow the `DELE`, `RNFR` and `RNTO` -commands, which standard FTP servers do. +it into place, so a failed upload keeps the latest dump. The server must +allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers +do. 1. In `.env`, set the fetch source and the server: diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 16ba2bf9d5..c37cdc1565 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -549,7 +549,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | -| `VORTEX_PUSH_DB_FILE_PREFIX` | Prefix for the names of the pushed database dump files.

Use a different prefix in each place that pushes to the same storage, such as 'gha-' and 'circleci-', to tell their dumps apart. A fetch reads the prefixed name only when its remote file name includes the prefix. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_FILE_SUFFIX` | Suffix for the names of the pushed database dump files.

Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. Use a different suffix in each place that pushes to the same storage. A fetch reads the suffixed name only when its remote file name includes it. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | @@ -567,6 +567,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_REMOTE_FILE` | Remote database dump file name.

Defaults to the remote file name the database is fetched from, then to the database dump file name. | `${VORTEX_PUSH_DB_S3_DB_FILE}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_SECRET_KEY` | AWS secret key. | `${S3_SECRET_KEY}` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | +| `VORTEX_PUSH_DB_TIMESTAMPED_COPY` | Also push a copy of the database dump file named with a UTC timestamp.

Set to `1` to keep a history of the pushed dumps. The fetch never reads the timestamped copies. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | From d99cecebd2fcc4f79c5edc775544f3f2a78e3c97 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:21:11 +1100 Subject: [PATCH 17/40] Updated snapshots. --- .../handler_process/db_fetch_source_ftp/.env | 15 ++++++++++----- .../handler_process/db_fetch_source_s3/.env | 15 ++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index a6c57df4f1..8d266c9412 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,12 +180,27 @@ +@@ -180,13 +180,32 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -23,14 +23,19 @@ -VORTEX_FETCH_DB_URL= +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source, keeping the dump it replaces as the previous one. The fetch -+# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -+# for url. Add them, and keep secrets in CI. ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 ++ # Environment to fetch the database from. # -@@ -254,17 +269,3 @@ + # Applies to hosting environments. +@@ -254,17 +273,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 63867d4b6e..aa442ec9c3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,12 +180,24 @@ +@@ -180,13 +180,29 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -20,14 +20,19 @@ -VORTEX_FETCH_DB_URL= +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source, keeping the dump it replaces as the previous one. The fetch -+# script of this source reads its own variables, such as VORTEX_FETCH_DB_URL -+# for url. Add them, and keep secrets in CI. ++# fetch source as the latest dump. The fetch script of this source reads its ++# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep ++# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url ++# Also push a copy of each dump named with a UTC timestamp, such as ++# db_20261111-180000.sql, to keep a history of the pushed dumps. ++# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 ++ # Environment to fetch the database from. # -@@ -254,17 +266,3 @@ + # Applies to hosting environments. +@@ -254,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" From 60571a563089f857fdf8fed2def076ab81f17fc4 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 15:52:16 +1100 Subject: [PATCH 18/40] Addressed code review: sent an empty length on the S3 copy request and read the fetch directory for the S3 push. --- .vortex/tooling/src/vortex-push-db-s3 | 9 +++++++-- .vortex/tooling/tests/unit/push-db-s3.bats | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index c58857fa4a..0a9ce5ea95 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -53,8 +53,9 @@ VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" # Directory with database dump file. _v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" -_vs="VORTEX_DB${_db_index}_DIR" -VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${VORTEX_DB_DIR:-./.data}}}" +_vs="VORTEX_FETCH_DB${_db_index}_DIR" +_vss="VORTEX_DB${_db_index}_DIR" +VORTEX_PUSH_DB_S3_DB_DIR="${!_v:-${!_vs:-${!_vss:-${VORTEX_DB_DIR:-./.data}}}}" # Database dump file name. # @@ -175,8 +176,12 @@ ${canonical_request_hash}" signature=$(aws_signature "${VORTEX_PUSH_DB_S3_SECRET_KEY}" "${date_short}" "${VORTEX_PUSH_DB_S3_REGION}" "s3" "${string_to_sign}") local args=(--silent --show-error --location --proto-redir "=https" --connect-timeout "${VORTEX_PUSH_DB_S3_CONNECT_TIMEOUT:-10}" --max-time "${VORTEX_PUSH_DB_S3_MAX_TIME:-300}" --request PUT --write-out "\n%{http_code}") + # curl sends no length for a PUT without a body, and S3 rejects such a + # request. if [ -n "${upload_file}" ]; then args+=(--upload-file "${upload_file}") + else + args+=(--header "Content-Length: 0") fi local header diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index f9c066e57f..95a5ca50f8 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -579,6 +579,8 @@ setup_s3_push() { copy_args="$(mock_get_call_args "${mock_curl}" 2)" assert_string_contains "${copy_args}" "--header x-amz-copy-source: /test-bucket/backups/db.sql" assert_string_not_contains "${copy_args}" "--upload-file" + assert_string_contains "${copy_args}" "--header Content-Length: 0" + assert_string_not_contains "$(mock_get_call_args "${mock_curl}" 1)" "Content-Length: 0" [[ ${copy_args} =~ https://test-bucket\.s3\.ap-southeast-2\.amazonaws\.com/backups/db_[0-9]{8}-[0-9]{6}\.sql ]] rm -rf .data @@ -629,6 +631,10 @@ setup_s3_push() { export VORTEX_FETCH_DB2_S3_REGION="eu-west-1" export VORTEX_FETCH_DB2_S3_PREFIX="" export VORTEX_FETCH_DB2_FILE="db2.sql" + # The dump directory follows the directory the database is fetched to. + mkdir -p .data/db2 + mv .data/db2.sql .data/db2/db2.sql + export VORTEX_FETCH_DB2_DIR=".data/db2" export VORTEX_PUSH_DB_FILE_SUFFIX="_circleci" export VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 @@ -639,7 +645,7 @@ setup_s3_push() { run .vortex/tooling/src/vortex-push-db-s3 assert_success assert_output_contains "[INFO] Started database 2 dump push to S3." - assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Local file: .data/db2/db2.sql" assert_output_contains "Remote file: db2_circleci.sql" assert_output_contains "S3 bucket: db2-bucket" assert_output_contains "[ OK ] Copied the database dump file to db2_circleci_" From 8f233b5714d250b68c8bc07fceee91fc84dd5dab Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:18:02 +1100 Subject: [PATCH 19/40] Addressed code review: documented setting separate push and fetch filenames for a suffixed dump. --- .../content/development/database-storage.mdx | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index b4f9044238..0269319729 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -87,8 +87,10 @@ refreshes the store from the upstream source: └── db_20261111-180000.sql ``` -`db.sql` always holds the latest dump, and fetching downloads it. By -default, it's the only file a push creates. To keep a history of the +`db.sql` always holds the latest dump, and fetching downloads it. With a +file suffix, the latest dump is the suffixed file, such as `db_gha.sql` +(see [Tell pushes apart](#tell-pushes-apart)). By default, it's the only +file a push creates. To keep a history of the pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push then also creates a copy named with its UTC time, identical to `db.sql` at that moment. Nothing removes the copies, so clear out the old ones @@ -203,8 +205,23 @@ fails for a larger dump. On FTP, the timestamped copy is a second upload. `VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a -different suffix in each place that pushes to the same store. A fetch reads -the suffixed file only when its remote filename includes the suffix. +different suffix in each place that pushes to the same store. + +The push takes its remote filename from the fetch, then adds the suffix. To +fetch a suffixed dump from an FTP store, set both filenames, so the suffix +appears once: + +```shell +# The push adds the suffix to this name. +VORTEX_PUSH_DB_FTP_FILE=your_site/db.sql +# The fetch reads the suffixed name. +VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql +``` + +For S3, use the push remote filename `VORTEX_PUSH_DB_S3_REMOTE_FILE` in +place of `VORTEX_PUSH_DB_FTP_FILE`. The S3 fetch names the local dump after +the remote file, so fetching a suffixed dump from S3 also renames the local +dump. ## What's next From beba57c80151ba7c028002ff98fa926f60a03b04 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:38:53 +1100 Subject: [PATCH 20/40] Addressed code review: documented renaming the dump with 'VORTEX_DB_FILE' for a suffixed S3 fetch. --- .../docs/content/development/database-storage.mdx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 0269319729..f40871cbe2 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -218,10 +218,17 @@ VORTEX_PUSH_DB_FTP_FILE=your_site/db.sql VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql ``` -For S3, use the push remote filename `VORTEX_PUSH_DB_S3_REMOTE_FILE` in -place of `VORTEX_PUSH_DB_FTP_FILE`. The S3 fetch names the local dump after -the remote file, so fetching a suffixed dump from S3 also renames the local -dump. +The S3 fetch names the local dump after the remote file. To fetch a +suffixed dump from an S3 store, rename the dump everywhere with +`VORTEX_DB_FILE`, which the fetch, the refresh and the import all read. +Then set the push remote filename to the unsuffixed name: + +```shell +# The fetch, the refresh and the import use the suffixed name. +VORTEX_DB_FILE=db_gha.sql +# The push adds the suffix to this name. +VORTEX_PUSH_DB_S3_REMOTE_FILE=db.sql +``` ## What's next From 7cfda42c4b8563d0a9e622483018679f69981b10 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 16:43:20 +1100 Subject: [PATCH 21/40] Reworded the suffixed S3 fetch note to pass the docs spellcheck. --- .vortex/docs/content/development/database-storage.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index f40871cbe2..9b3102336c 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -221,7 +221,7 @@ VORTEX_FETCH_DB_FTP_FILE=your_site/db_gha.sql The S3 fetch names the local dump after the remote file. To fetch a suffixed dump from an S3 store, rename the dump everywhere with `VORTEX_DB_FILE`, which the fetch, the refresh and the import all read. -Then set the push remote filename to the unsuffixed name: +Then set the push remote filename to the name without the suffix: ```shell # The fetch, the refresh and the import use the suffixed name. From a6fa78dc4ab458be04319ac0b27aa0e9110ea36e Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Wed, 11 Nov 2026 23:15:52 +1100 Subject: [PATCH 22/40] Added a playground script to fetch the database dump from FTP. --- .../tooling/playground/try-fetch-db-ftp.sh | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100755 .vortex/tooling/playground/try-fetch-db-ftp.sh diff --git a/.vortex/tooling/playground/try-fetch-db-ftp.sh b/.vortex/tooling/playground/try-fetch-db-ftp.sh new file mode 100755 index 0000000000..6772da9ec5 --- /dev/null +++ b/.vortex/tooling/playground/try-fetch-db-ftp.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +## +# Manual test script for FTP database fetch. +# +# Downloads the latest database dump file from an FTP server using +# fetch-db-ftp, for example with a read-only Storage Box subaccount. +# +# Usage: +# export FTP_HOST="ftp.example.com" +# export FTP_PORT="21" +# export FTP_USER="your-read-user" +# export FTP_PASS="your-read-password" +# export FTP_FILE="path/to/db.sql" +# ./try-fetch-db-ftp.sh + +set -eu + +# Source secrets if available. +if [ -f "$HOME/.profile.secrets" ]; then + # shellcheck disable=SC1091 + . "$HOME/.profile.secrets" +fi + +# Get the directory of this script and navigate to project root. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# FTP credentials and settings. +FTP_HOST="${FTP_HOST:-}" +FTP_PORT="${FTP_PORT:-21}" +FTP_USER="${FTP_USER:-}" +FTP_PASS="${FTP_PASS:-}" +FTP_FILE="${FTP_FILE:-db.sql}" + +# Validate required variables. +for name in FTP_HOST FTP_USER FTP_PASS; do + if [ -z "${!name}" ]; then + echo "Error: ${name} environment variable is required" + echo "Usage: export ${name}=\"...\" && $0" + exit 1 + fi +done + +db_dir="${VORTEX_DB_DIR:-./.data}" +db_file="${VORTEX_DB_FILE:-db.sql}" + +echo "Testing FTP database fetch..." +echo "" +echo "FTP host: ${FTP_HOST}:${FTP_PORT}" +echo "FTP file: ${FTP_FILE}" +echo "Local file: ${db_dir}/${db_file}" +echo "" + +cd "${PROJECT_ROOT}" || exit 1 + +export VORTEX_FETCH_DB_FTP_HOST="${FTP_HOST}" +export VORTEX_FETCH_DB_FTP_PORT="${FTP_PORT}" +export VORTEX_FETCH_DB_FTP_USER="${FTP_USER}" +export VORTEX_FETCH_DB_FTP_PASS="${FTP_PASS}" +export VORTEX_FETCH_DB_FTP_FILE="${FTP_FILE}" + +./vendor/drevops/vortex-tooling/src/vortex-fetch-db-ftp + +echo "" +echo "Fetch complete!" From 93b713c9733c2cb20520d2efaa1ea040134edc16 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 09:57:10 +1100 Subject: [PATCH 23/40] Added the 'VORTEX_FETCH_DB_FTP_DIR' remote directory for the FTP fetch and push, and pushed Vortex's own demo dumps into per-provider directories. --- .circleci/config.yml | 6 +- .env | 10 +++ .github/workflows/build-test-deploy.yml | 6 +- .vortex/tooling/src/vortex-fetch-db-ftp | 30 ++++++- .vortex/tooling/src/vortex-push-db-ftp | 25 +++++- .vortex/tooling/tests/unit/fetch-db-ftp.bats | 82 ++++++++++++++++++ .vortex/tooling/tests/unit/push-db-ftp.bats | 88 +++++++++++++++++++- 7 files changed, 238 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7985a20971..61ee3535c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -416,9 +416,9 @@ jobs: # Refresh the database dumps in the fetch sources that are dump stores. VORTEX_PUSH_DB_PROCEED: 1 #;< VORTEX_DEV - # Tell this provider's demo database dumps apart on the test FTP server, - # with a timestamped copy of each push. - VORTEX_PUSH_DB_FILE_SUFFIX: _circleci + # Tell this provider's demo database dumps apart on the test FTP server + # by directory, with a timestamped copy of each push. + VORTEX_PUSH_DB_FTP_DIR: circleci VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV # Do not build the Drupal front-end. diff --git a/.env b/.env index 661a9a946b..3d05a84c19 100644 --- a/.env +++ b/.env @@ -244,6 +244,11 @@ VORTEX_FETCH_DB_FTP_HOST= # Database dump FTP port. VORTEX_FETCH_DB_FTP_PORT=21 +# Database dump FTP directory. +# +# Leave empty for the root directory of the FTP account. +VORTEX_FETCH_DB_FTP_DIR= + # Database dump FTP file name. VORTEX_FETCH_DB_FTP_FILE=db.sql #;> DB_FETCH_SOURCE_FTP @@ -334,6 +339,11 @@ VORTEX_FETCH_DB2_FTP_HOST= # Second database dump FTP port. VORTEX_FETCH_DB2_FTP_PORT=21 +# Second database dump FTP directory. +# +# Leave empty for the root directory of the FTP account. +VORTEX_FETCH_DB2_FTP_DIR= + # Second database dump FTP file name. VORTEX_FETCH_DB2_FTP_FILE=db2.sql #;> MIGRATION_DB_FETCH_SOURCE_FTP diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index d126090f7d..8c528e0d75 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -242,15 +242,15 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, under - # a suffix that tells this provider's dumps apart, with a timestamped + # FTP server that receives the demo database on the nightly run, in a + # directory that tells this provider's dumps apart, with a timestamped # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_DIR: ${{ vars.VORTEX_PUSH_DB_FTP_DIR || 'gha' }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_SUFFIX: _gha VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV diff --git a/.vortex/tooling/src/vortex-fetch-db-ftp b/.vortex/tooling/src/vortex-fetch-db-ftp index c7c190d193..2119b31768 100755 --- a/.vortex/tooling/src/vortex-fetch-db-ftp +++ b/.vortex/tooling/src/vortex-fetch-db-ftp @@ -35,6 +35,13 @@ VORTEX_FETCH_DB_FTP_PORT="${!_v:-}" _v="VORTEX_FETCH_DB${_db_index}_FTP_FILE" VORTEX_FETCH_DB_FTP_FILE="${!_v:-}" +# The directory on the FTP server that holds the file. +# +# Joined in front of the file name. Leading and trailing slashes are optional. +# Empty for the root directory of the FTP account. +_v="VORTEX_FETCH_DB${_db_index}_FTP_DIR" +VORTEX_FETCH_DB_FTP_DIR="${!_v:-}" + # Directory with database dump file. _v="VORTEX_FETCH_DB${_db_index}_FTP_DB_DIR" _vs="VORTEX_FETCH_DB${_db_index}_DIR" @@ -57,6 +64,19 @@ pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START) fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } # @formatter:on +# Print a remote directory with no leading slash and 1 trailing slash, so it +# joins a file name. Prints nothing for an empty or root directory. +ftp_dir() { + local dir="${1}" + + while [[ ${dir} == /* ]]; do dir="${dir#/}"; done + while [[ ${dir} == */ ]]; do dir="${dir%/}"; done + + if [ -n "${dir}" ]; then + printf '%s/' "${dir}" + fi +} + # shellcheck disable=SC2043 for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is not available."; done @@ -66,10 +86,18 @@ for cmd in curl; do command -v "${cmd}" >/dev/null || fail "Command ${cmd} is no [ -z "${VORTEX_FETCH_DB_FTP_PORT}" ] && fail "Missing required value for VORTEX_FETCH_DB_FTP_PORT." [ -z "${VORTEX_FETCH_DB_FTP_FILE}" ] && fail "Missing required value for VORTEX_FETCH_DB_FTP_FILE." +remote_dir="$(ftp_dir "${VORTEX_FETCH_DB_FTP_DIR}")" +remote_file="${VORTEX_FETCH_DB_FTP_FILE}" +if [ -n "${remote_dir}" ]; then + remote_file="${remote_dir}${remote_file#/}" +fi + info "Started database dump fetch from FTP." +note "Remote file: ${remote_file}" + mkdir -p "${VORTEX_FETCH_DB_FTP_DB_DIR}" -curl -u "${VORTEX_FETCH_DB_FTP_USER}":"${VORTEX_FETCH_DB_FTP_PASS}" "ftp://${VORTEX_FETCH_DB_FTP_HOST}:${VORTEX_FETCH_DB_FTP_PORT}/${VORTEX_FETCH_DB_FTP_FILE}" -o "${VORTEX_FETCH_DB_FTP_DB_DIR}/${VORTEX_FETCH_DB_FTP_DB_FILE}" +curl -u "${VORTEX_FETCH_DB_FTP_USER}":"${VORTEX_FETCH_DB_FTP_PASS}" "ftp://${VORTEX_FETCH_DB_FTP_HOST}:${VORTEX_FETCH_DB_FTP_PORT}/${remote_file}" -o "${VORTEX_FETCH_DB_FTP_DB_DIR}/${VORTEX_FETCH_DB_FTP_DB_FILE}" pass "Finished database dump fetch from FTP." diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp index f2abb91e23..0cfa7ce2ed 100755 --- a/.vortex/tooling/src/vortex-push-db-ftp +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -51,6 +51,16 @@ _v="VORTEX_PUSH_DB${_db_index}_FTP_FILE" _vs="VORTEX_FETCH_DB${_db_index}_FTP_FILE" VORTEX_PUSH_DB_FTP_FILE="${!_v:-${!_vs:-}}" +# The directory on the FTP server to push the file to. +# +# Joined in front of the file name. Leading and trailing slashes are optional. +# Defaults to the directory the database is fetched from. Use a different +# directory in each place that pushes to the same storage, such as 'gha' and +# 'circleci'. +_v="VORTEX_PUSH_DB${_db_index}_FTP_DIR" +_vs="VORTEX_FETCH_DB${_db_index}_FTP_DIR" +VORTEX_PUSH_DB_FTP_DIR="${!_v:-${!_vs:-}}" + # Directory with database dump file. _v="VORTEX_PUSH_DB${_db_index}_FTP_DB_DIR" _vs="VORTEX_FETCH_DB${_db_index}_DIR" @@ -88,6 +98,19 @@ pass() { _d=""; [ -n "${_TASK_START:-}" ] && _d=" ($(($(date +%s) - _TASK_START) fail() { [ "${TERM:-}" != "dumb" ] && tput colors >/dev/null 2>&1 && printf "\033[31m[FAIL] %s\033[0m\n" "${1}" || printf "[FAIL] %s\n" "${1}"; exit "${2:-1}"; } # @formatter:on +# Print a remote directory with no leading slash and 1 trailing slash, so it +# joins a file name. Prints nothing for an empty or root directory. +ftp_dir() { + local dir="${1}" + + while [[ ${dir} == /* ]]; do dir="${dir#/}"; done + while [[ ${dir} == */ ]]; do dir="${dir%/}"; done + + if [ -n "${dir}" ]; then + printf '%s/' "${dir}" + fi +} + # Print a value quoted for a curl config file. curl_config_quote() { local value="${1//\\/\\\\}" @@ -122,7 +145,7 @@ local_file="${VORTEX_PUSH_DB_FTP_DB_DIR}/${VORTEX_PUSH_DB_FTP_DB_FILE}" [ ! -f "${local_file}" ] && fail "Database dump file ${local_file} does not exist." remote_dir="" -remote_name="${VORTEX_PUSH_DB_FTP_FILE#/}" +remote_name="$(ftp_dir "${VORTEX_PUSH_DB_FTP_DIR}")${VORTEX_PUSH_DB_FTP_FILE#/}" if [[ ${remote_name} == */* ]]; then remote_dir="${remote_name%/*}/" remote_name="${remote_name##*/}" diff --git a/.vortex/tooling/tests/unit/fetch-db-ftp.bats b/.vortex/tooling/tests/unit/fetch-db-ftp.bats index 6429706d5b..116b8010f6 100644 --- a/.vortex/tooling/tests/unit/fetch-db-ftp.bats +++ b/.vortex/tooling/tests/unit/fetch-db-ftp.bats @@ -129,3 +129,85 @@ load ../_helper.bash popd >/dev/null } + +@test "fetch-db-ftp: Join the remote directory to the file name with any slashes" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_FTP_USER="testuser" + export VORTEX_FETCH_DB_FTP_PASS="testpass" + export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB_FTP_PORT="21" + export VORTEX_FETCH_DB_FTP_FILE="db.sql" + export VORTEX_FETCH_DB_FTP_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + + call=0 + for dir in "nightly/gha" "/nightly/gha" "nightly/gha/" "/nightly/gha/" "//nightly/gha//"; do + export VORTEX_FETCH_DB_FTP_DIR="${dir}" + call=$((call + 1)) + + run .vortex/tooling/src/vortex-fetch-db-ftp + assert_success + assert_output_contains "Remote file: nightly/gha/db.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" "${call}")" "ftp://ftp.example.com:21/nightly/gha/db.sql " + done + + # A leading slash on the file name joins without a double slash. + export VORTEX_FETCH_DB_FTP_DIR="nightly" + export VORTEX_FETCH_DB_FTP_FILE="/db.sql" + run .vortex/tooling/src/vortex-fetch-db-ftp + assert_success + assert_string_contains "$(mock_get_call_args "${mock_curl}" 6)" "ftp://ftp.example.com:21/nightly/db.sql " + + popd >/dev/null +} + +@test "fetch-db-ftp: Fetch from the root directory without a remote directory" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_FETCH_DB_FTP_USER="testuser" + export VORTEX_FETCH_DB_FTP_PASS="testpass" + export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB_FTP_PORT="21" + export VORTEX_FETCH_DB_FTP_FILE="backup/db.sql" + export VORTEX_FETCH_DB_FTP_DB_DIR=".data" + + mock_curl=$(mock_command "curl") + + call=0 + for dir in "" "/" "//"; do + export VORTEX_FETCH_DB_FTP_DIR="${dir}" + call=$((call + 1)) + + run .vortex/tooling/src/vortex-fetch-db-ftp + assert_success + assert_output_contains "Remote file: backup/db.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" "${call}")" "ftp://ftp.example.com:21/backup/db.sql " + done + + popd >/dev/null +} + +@test "fetch-db-ftp: Use the indexed remote directory with VORTEX_DB_INDEX" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_FTP_USER="testuser" + export VORTEX_FETCH_DB2_FTP_PASS="testpass" + export VORTEX_FETCH_DB2_FTP_HOST="ftp.example.com" + export VORTEX_FETCH_DB2_FTP_PORT="21" + export VORTEX_FETCH_DB2_FTP_FILE="db2.sql" + export VORTEX_FETCH_DB2_FTP_DIR="/migration/" + export VORTEX_FETCH_DB_FTP_DIR="primary" + export VORTEX_FETCH_DB2_DIR=".data" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-fetch-db-ftp + assert_success + assert_output_contains "Remote file: migration/db2.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp.example.com:21/migration/db2.sql " + + popd >/dev/null +} diff --git a/.vortex/tooling/tests/unit/push-db-ftp.bats b/.vortex/tooling/tests/unit/push-db-ftp.bats index efc6ad44eb..bf138d08a9 100644 --- a/.vortex/tooling/tests/unit/push-db-ftp.bats +++ b/.vortex/tooling/tests/unit/push-db-ftp.bats @@ -13,7 +13,7 @@ setup_ftp_fetch_values() { export VORTEX_FETCH_DB_FTP_HOST="ftp.example.com" export VORTEX_FETCH_DB_FTP_PORT="21" export VORTEX_FETCH_DB_FTP_FILE="db.sql" - unset VORTEX_PUSH_DB_FTP_USER VORTEX_PUSH_DB_FTP_PASS VORTEX_PUSH_DB_FTP_HOST VORTEX_PUSH_DB_FTP_PORT VORTEX_PUSH_DB_FTP_FILE VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY + unset VORTEX_PUSH_DB_FTP_USER VORTEX_PUSH_DB_FTP_PASS VORTEX_PUSH_DB_FTP_HOST VORTEX_PUSH_DB_FTP_PORT VORTEX_PUSH_DB_FTP_FILE VORTEX_PUSH_DB_FTP_DIR VORTEX_FETCH_DB_FTP_DIR VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY } @test "push-db-ftp: Upload only the latest dump by default" { @@ -260,3 +260,89 @@ setup_ftp_fetch_values() { popd >/dev/null } + +@test "push-db-ftp: Push to the remote directory with any slashes" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + + call=0 + for dir in "nightly/gha" "/nightly/gha" "nightly/gha/" "/nightly/gha/" "//nightly/gha//"; do + export VORTEX_PUSH_DB_FTP_DIR="${dir}" + call=$((call + 1)) + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: nightly/gha/db.sql" + + # The rename commands run in the remote directory, so they use bare names. + args="$(mock_get_call_args "${mock_curl}" "${call}")" + assert_string_contains "${args}" "--quote -RNTO db.sql" + assert_string_contains "${args}" "ftp://ftp.example.com:21/nightly/gha/db.sql.uploading" + done + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to the directory the database is fetched from" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + export VORTEX_FETCH_DB_FTP_DIR="/shared/" + export VORTEX_FETCH_DB_FTP_FILE="site/db.sql" + + mock_curl=$(mock_command "curl") + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + # The directory goes in front of any directories in the file name. + assert_output_contains "Remote file: shared/site/db.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://ftp.example.com:21/shared/site/db.sql.uploading" + + # A push directory overrides the fetch directory. + export VORTEX_PUSH_DB_FTP_DIR="circleci" + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: circleci/site/db.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" 2)" "ftp://ftp.example.com:21/circleci/site/db.sql.uploading" + + rm -rf .data + + popd >/dev/null +} + +@test "push-db-ftp: Push to the root directory without a remote directory" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "database content" >.data/db.sql + + setup_ftp_fetch_values + + mock_curl=$(mock_command "curl") + + call=0 + for dir in "" "/" "//"; do + export VORTEX_PUSH_DB_FTP_DIR="${dir}" + call=$((call + 1)) + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + assert_output_contains "Remote file: db.sql" + assert_string_contains "$(mock_get_call_args "${mock_curl}" "${call}")" "ftp://ftp.example.com:21/db.sql.uploading" + done + + rm -rf .data + + popd >/dev/null +} From 93b64a33226d7aadc362ff7bd04fca31761683d2 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 10:00:37 +1100 Subject: [PATCH 24/40] Added 'VORTEX_PUSH_DB_DESTINATIONS' to push the database dump to several destinations, and ran the push steps for every fetch source. --- .circleci/config.yml | 16 +- .circleci/vortex-test-common.yml | 16 +- .env | 14 +- .github/workflows/build-test-deploy.yml | 16 +- .github/workflows/vortex-test-didi.yml | 44 +++-- .../Prompts/Handlers/DatabaseFetchSource.php | 4 +- .../Prompts/Handlers/MigrationFetchSource.php | 4 +- ...MigrationFetchSourceHandlerProcessTest.php | 4 +- .vortex/tooling/src/vortex-push-db | 55 +++++-- .vortex/tooling/src/vortex-refresh-db | 24 +-- .vortex/tooling/tests/_helper.bash | 1 + .vortex/tooling/tests/unit/push-db.bats | 154 +++++++++++++----- .vortex/tooling/tests/unit/refresh-db.bats | 29 +++- 13 files changed, 258 insertions(+), 123 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61ee3535c0..26836dc8d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -363,23 +363,21 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY - #;< DB_PUSH - # Push the freshly-exported sanitized database dump to the database fetch - # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the - # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). - run: name: Push DB command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi #;> VORTEX_DEV no_output_timeout: 30m - #;> DB_PUSH - #;< MIGRATION_DB_PUSH - # Push the freshly-fetched migration database dump to its fetch source, + #;< MIGRATION + # Push the freshly-fetched migration database dump to its push destinations, # next to the primary dump. No-op unless the push was requested. - run: name: Push migration DB @@ -387,7 +385,7 @@ jobs: [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db no_output_timeout: 30m - #;> MIGRATION_DB_PUSH + #;> MIGRATION - save_cache: # Save cache per default branch and the timestamp. diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 71b2bbfeb1..36d2f77968 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -183,23 +183,21 @@ jobs: no_output_timeout: 30m #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY - #;< DB_PUSH - # Push the freshly-exported sanitized database dump to the database fetch - # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the - # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). - run: name: Push DB command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi #;> VORTEX_DEV no_output_timeout: 30m - #;> DB_PUSH - #;< MIGRATION_DB_PUSH - # Push the freshly-fetched migration database dump to its fetch source, + #;< MIGRATION + # Push the freshly-fetched migration database dump to its push destinations, # next to the primary dump. No-op unless the push was requested. - run: name: Push migration DB @@ -207,7 +205,7 @@ jobs: [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db no_output_timeout: 30m - #;> MIGRATION_DB_PUSH + #;> MIGRATION - save_cache: # Save cache per default branch and the timestamp. diff --git a/.env b/.env index 3d05a84c19..2ca87bb620 100644 --- a/.env +++ b/.env @@ -272,11 +272,18 @@ VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 # own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep # secrets in CI. # VORTEX_REFRESH_DB_SOURCE=url +#;> DB_PUSH + +# Destinations the nightly CI job pushes the sanitized database dump to: +# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source +# when it is a dump store. Each destination reads its own variables, such as +# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 +# source. +# VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 # Also push a copy of each dump named with a UTC timestamp, such as # db_20261111-180000.sql, to keep a history of the pushed dumps. # VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 -#;> DB_PUSH # Environment to fetch the database from. # @@ -367,6 +374,11 @@ VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 # VORTEX_REFRESH_DB2_SOURCE=url #;> MIGRATION_DB_PUSH +# Destinations the nightly CI job pushes the second database dump to, as +# fetched: ftp, s3 or both. Defaults to the second database fetch source when +# it is a dump store. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + # Environment to fetch the second database from. # # Applies to hosting environments. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 8c528e0d75..c3c0f7b770 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -373,29 +373,27 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY - #;< DB_PUSH - # Push the freshly-exported sanitized database dump to the database fetch - # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the - # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). - name: Push DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi #;> VORTEX_DEV timeout-minutes: 30 - #;> DB_PUSH - #;< MIGRATION_DB_PUSH - # Push the freshly-fetched migration database dump to its fetch source, + #;< MIGRATION + # Push the freshly-fetched migration database dump to its push destinations, # next to the primary dump. No-op unless the push was requested. - name: Push migration DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db timeout-minutes: 30 - #;> MIGRATION_DB_PUSH + #;> MIGRATION # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index b2b4d23cb4..e5b891a370 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -81,15 +81,15 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, under - # a suffix that tells this provider's dumps apart, with a timestamped + # FTP server that receives the demo database on the nightly run, in a + # directory that tells this provider's dumps apart, with a timestamped # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_DIR: ${{ vars.VORTEX_PUSH_DB_FTP_DIR || 'gha' }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_SUFFIX: _gha VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: url @@ -221,29 +221,27 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY - #;< DB_PUSH - # Push the freshly-exported sanitized database dump to the database fetch - # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the - # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). - name: Push DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi #;> VORTEX_DEV timeout-minutes: 30 - #;> DB_PUSH - #;< MIGRATION_DB_PUSH - # Push the freshly-fetched migration database dump to its fetch source, + #;< MIGRATION + # Push the freshly-fetched migration database dump to its push destinations, # next to the primary dump. No-op unless the push was requested. - name: Push migration DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db timeout-minutes: 30 - #;> MIGRATION_DB_PUSH + #;> MIGRATION # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -657,15 +655,15 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, under - # a suffix that tells this provider's dumps apart, with a timestamped + # FTP server that receives the demo database on the nightly run, in a + # directory that tells this provider's dumps apart, with a timestamped # copy of each push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} VORTEX_PUSH_DB_FTP_PASS: ${{ secrets.VORTEX_PUSH_DB_FTP_PASS }} + VORTEX_PUSH_DB_FTP_DIR: ${{ vars.VORTEX_PUSH_DB_FTP_DIR || 'gha' }} VORTEX_PUSH_DB_FTP_FILE: ${{ vars.VORTEX_PUSH_DB_FTP_FILE || 'db.sql' }} - VORTEX_PUSH_DB_FILE_SUFFIX: _gha VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV VORTEX_FETCH_DB_SOURCE: container_registry @@ -815,29 +813,27 @@ jobs: timeout-minutes: 30 #;> DB_FETCH_SOURCE_CONTAINER_REGISTRY - #;< DB_PUSH - # Push the freshly-exported sanitized database dump to the database fetch - # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the - # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). - name: Push DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_FETCH_DB_SOURCE=ftp VORTEX_REFRESH_DB_SOURCE=url ./vendor/bin/vortex-push-db; fi + if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi #;> VORTEX_DEV timeout-minutes: 30 - #;> DB_PUSH - #;< MIGRATION_DB_PUSH - # Push the freshly-fetched migration database dump to its fetch source, + #;< MIGRATION + # Push the freshly-fetched migration database dump to its push destinations, # next to the primary dump. No-op unless the push was requested. - name: Push migration DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db timeout-minutes: 30 - #;> MIGRATION_DB_PUSH + #;> MIGRATION # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index f5abc209ff..baf3d8b360 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -151,8 +151,8 @@ public function process(): void { File::removeTokenAsync('DB_IN_IMAGE'); } - // DB_PUSH wraps the nightly push of the sanitized dump, which only an - // 'ftp' or 's3' source accepts. + // DB_PUSH wraps the upstream source of a dump store, which only an + // 'ftp' or 's3' source can be. if (!in_array($v, [self::FTP, self::S3], TRUE)) { File::removeTokenAsync('DB_PUSH'); } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index 53c2b31b7c..4307d6847d 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -145,8 +145,8 @@ public function process(): void { } } - // MIGRATION_DB_PUSH wraps the nightly push of the migration dump, which - // only an 'ftp' or 's3' source accepts. + // MIGRATION_DB_PUSH wraps the upstream source of a dump store, which + // only an 'ftp' or 's3' source can be. if (!in_array($v, [self::FTP, self::S3], TRUE)) { File::removeTokenAsync('MIGRATION_DB_PUSH'); } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index 9c128f4a8a..2ae12c8fdd 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -27,7 +27,9 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-refresh-db'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_IMAGE_BASE'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'Deploy DB image'); - $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); + // Any migration source can push to its own destinations. + $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_REFRESH_DB2_SOURCE=url'); }), ]; yield 'migration_fetch_source_ftp' => [ diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db index 9a36c9629f..2035c75a87 100755 --- a/.vortex/tooling/src/vortex-push-db +++ b/.vortex/tooling/src/vortex-push-db @@ -4,10 +4,11 @@ # # This is a router script to call relevant scripts based on type. # -# The database dump file is pushed to the storage the database is fetched -# from, set in VORTEX_FETCH_DB_SOURCE. Only an 'ftp' or 's3' source with an -# upstream source in VORTEX_REFRESH_DB_SOURCE accepts a push (a dump store); -# the push is skipped for other sources. +# The database dump file is pushed to every destination in +# VORTEX_PUSH_DB_DESTINATIONS, such as 'ftp,s3'. When it is empty and the +# 'ftp' or 's3' fetch source has an upstream source in +# VORTEX_REFRESH_DB_SOURCE (a dump store), the dump is pushed to the fetch +# source. The database is always fetched from 1 source. # # With VORTEX_DB_INDEX set (e.g., "2" for the migration database), the # indexed database pushes its own dump file with the indexed variables, so @@ -32,7 +33,15 @@ _db_index="${VORTEX_DB_INDEX:-}" # and never from a feature branch build. VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" -# Database fetch source, which is also the push destination. +# Destinations to push the database dump file to: 'ftp', 's3' or both, +# separated by commas or spaces. +# +# Defaults to the fetch source when it is a dump store. Each destination reads +# its own variables, such as VORTEX_PUSH_DB_FTP_HOST for 'ftp'. +_v="VORTEX_PUSH_DB${_db_index}_DESTINATIONS" +VORTEX_PUSH_DB_DESTINATIONS="${!_v:-}" + +# Database fetch source. _v="VORTEX_FETCH_DB${_db_index}_SOURCE" VORTEX_FETCH_DB_SOURCE="${!_v:-url}" @@ -54,20 +63,38 @@ info "Started database${_db_index:+ ${_db_index}} push." [ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 -case "${VORTEX_FETCH_DB_SOURCE}" in - ftp) push_script="vortex-push-db-ftp" ;; - s3) push_script="vortex-push-db-s3" ;; - *) pass "Skipped database push as the \"${VORTEX_FETCH_DB_SOURCE}\" database source in VORTEX_FETCH_DB${_db_index}_SOURCE does not accept a push. Use one of: ftp, s3." && exit 0 ;; -esac +# A dump store receives the dump that the nightly refresh exports. +if [ -z "${VORTEX_PUSH_DB_DESTINATIONS}" ] && [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then + case "${VORTEX_FETCH_DB_SOURCE}" in + ftp | s3) VORTEX_PUSH_DB_DESTINATIONS="${VORTEX_FETCH_DB_SOURCE}" ;; + esac +fi + +destinations=() +for destination in ${VORTEX_PUSH_DB_DESTINATIONS//,/ }; do + case "${destination}" in + ftp | s3) ;; + *) fail "Unsupported database push destination \"${destination}\" in VORTEX_PUSH_DB${_db_index}_DESTINATIONS. Use one or more of: ftp, s3." ;; + esac + + # Pushing to the fetch source replaces the dump it fetched with its own + # export, so the source must be a dump store refreshed from upstream. + if [ "${destination}" = "${VORTEX_FETCH_DB_SOURCE}" ] && [ -z "${VORTEX_REFRESH_DB_SOURCE}" ]; then + fail "Refused to push the database to the \"${destination}\" fetch source as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the push would replace the dump it fetched." + fi + + destinations+=("${destination}") +done -# Without an upstream source, the fetch source is where the dump comes from, -# and a push would replace that dump with its own export. -[ -z "${VORTEX_REFRESH_DB_SOURCE}" ] && pass "Skipped database push as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the ${VORTEX_FETCH_DB_SOURCE} source is not a dump store." && exit 0 +[ "${#destinations[@]}" -eq 0 ] && pass "Skipped database push as VORTEX_PUSH_DB${_db_index}_DESTINATIONS is empty and the \"${VORTEX_FETCH_DB_SOURCE}\" fetch source is not a dump store." && exit 0 # The pushed dump leaves the hosting and CI, so it must be sanitized. Only the # primary database is sanitized: an indexed database is pushed as fetched. [ -z "${_db_index}" ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ] && fail "Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." -env VORTEX_DB_INDEX="${_db_index}" "$(dirname "${BASH_SOURCE[0]}")/${push_script}" +for destination in "${destinations[@]}"; do + note "Pushing to the ${destination} destination." + env VORTEX_DB_INDEX="${_db_index}" "$(dirname "${BASH_SOURCE[0]}")/vortex-push-db-${destination}" +done pass "Finished database${_db_index:+ ${_db_index}} push." diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index f32cf74240..b679c3bdc0 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,10 +15,10 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # -# With an 'ftp' or 's3' fetch source that has an upstream source in -# VORTEX_REFRESH_DB_SOURCE (a dump store) and VORTEX_PUSH_DB_PROCEED set to 1, -# the database is sanitized after provisioning, so the exported dump can be -# pushed to the store with 'vortex-push-db'. +# With VORTEX_PUSH_DB_PROCEED set to 1 and a push destination, either in +# VORTEX_PUSH_DB_DESTINATIONS or an 'ftp' or 's3' fetch source with an +# upstream source in VORTEX_REFRESH_DB_SOURCE (a dump store), the database is +# sanitized after provisioning, so 'vortex-push-db' pushes a sanitized dump. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -108,17 +108,21 @@ fi rebuild_image=0 [ -n "${db_image}" ] && [ -n "${db_image_base}" ] && rebuild_image=1 -# A dump store receives the dump this refresh exports, so with a push +# A push destination receives the dump this refresh exports, so with a push # requested the exported dump must be sanitized. refresh_store=0 -if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then - case "${fetch_source}" in - ftp | s3) refresh_store=1 ;; - esac +if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + if [ -n "${VORTEX_PUSH_DB_DESTINATIONS:-}" ]; then + refresh_store=1 + elif [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then + case "${fetch_source}" in + ftp | s3) refresh_store=1 ;; + esac + fi fi if [ "${refresh_store}" -eq 1 ] && [ "${VORTEX_PROVISION_SANITIZE_DB_SKIP:-}" = "1" ]; then - fail "The database dump file pushed to ${fetch_source} must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + fail "The pushed database dump file must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." fi if [ "${rebuild_image}" -eq 1 ] && [ "${fetch_source}" = "container_registry" ]; then diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 824a8b09f1..9da6a39932 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -122,6 +122,7 @@ setup() { unset VORTEX_FETCH_DB_FORCE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED unset VORTEX_PUSH_DB_PROCEED + unset VORTEX_PUSH_DB_DESTINATIONS unset VORTEX_PUSH_DB_FILE_SUFFIX unset VORTEX_PUSH_DB_TIMESTAMPED_COPY diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats index dbc3d256f4..c83bbe8446 100644 --- a/.vortex/tooling/tests/unit/push-db.bats +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -10,13 +10,18 @@ load ../_helper.bash +# Replaces each push script with a stub that names the destination it serves. +stub_push_scripts() { + stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + stub_sibling "vortex-push-db-s3" 'Pushed to s3 with index [${VORTEX_DB_INDEX}].' +} + @test "push-db: Skip when the push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + stub_push_scripts - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_DESTINATIONS="ftp" for proceed in "" 0; do export VORTEX_PUSH_DB_PROCEED="${proceed}" @@ -25,21 +30,21 @@ load ../_helper.bash assert_success assert_output_contains "[INFO] Started database push." assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." - assert_output_not_contains "Pushed to ftp." + assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." done popd >/dev/null } -@test "push-db: Route each dump store source to its push script" { +@test "push-db: Push to the dump store fetch source by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' - stub_sibling "vortex-push-db-s3" 'Pushed to s3 with index [${VORTEX_DB_INDEX}].' + stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_REFRESH_DB_SOURCE="lagoon" + unset VORTEX_PUSH_DB_DESTINATIONS for db_source in ftp s3; do export VORTEX_FETCH_DB_SOURCE="${db_source}" @@ -58,15 +63,58 @@ load ../_helper.bash popd >/dev/null } -@test "push-db: Push an indexed database with its own variables" { +@test "push-db: Push to every destination in the list" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" 'Pushed to ftp with index [${VORTEX_DB_INDEX}].' + stub_push_scripts + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="lagoon" + unset VORTEX_REFRESH_DB_SOURCE + + for destinations in "ftp,s3" "ftp s3" " ftp , s3 "; do + export VORTEX_PUSH_DB_DESTINATIONS="${destinations}" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushing to the ftp destination." + assert_output_contains "Pushed to ftp with index []." + assert_output_contains "Pushing to the s3 destination." + assert_output_contains "Pushed to s3 with index []." + assert_output_contains "[ OK ] Finished database push." + done + + popd >/dev/null +} + +@test "push-db: Push to the dump store fetch source and another destination" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_push_scripts + + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_DESTINATIONS="ftp,s3" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushed to ftp with index []." + assert_output_contains "Pushed to s3 with index []." + + popd >/dev/null +} + +@test "push-db: Push an indexed database to its own destinations" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_push_scripts export VORTEX_DB_INDEX=2 export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_FETCH_DB2_SOURCE="ftp" - export VORTEX_REFRESH_DB2_SOURCE="url" + export VORTEX_FETCH_DB2_SOURCE="url" + export VORTEX_PUSH_DB2_DESTINATIONS="s3" + export VORTEX_PUSH_DB_DESTINATIONS="ftp" # The indexed database is pushed as fetched, so the sanitization setting # does not apply to it. export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 @@ -74,27 +122,30 @@ load ../_helper.bash run .vortex/tooling/src/vortex-push-db assert_success assert_output_contains "[INFO] Started database 2 push." - assert_output_contains "Pushed to ftp with index [2]." + assert_output_contains "Pushed to s3 with index [2]." + assert_output_not_contains "Pushed to ftp" assert_output_contains "[ OK ] Finished database 2 push." popd >/dev/null } -@test "push-db: Skip for a source that does not accept a push" { +@test "push-db: Skip without destinations" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" "Pushed to ftp." - stub_sibling "vortex-push-db-s3" "Pushed to s3." + stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" + unset VORTEX_PUSH_DB_DESTINATIONS - for db_source in url acquia lagoon container_registry none; do - export VORTEX_FETCH_DB_SOURCE="${db_source}" + # A fetch source that is not a dump store, or a dump store source with no + # upstream source, gives no destination. + for case in "url:lagoon" "lagoon:lagoon" "container_registry:lagoon" "ftp:" "s3:"; do + export VORTEX_FETCH_DB_SOURCE="${case%%:*}" + export VORTEX_REFRESH_DB_SOURCE="${case#*:}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "[ OK ] Skipped database push as the \"${db_source}\" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3." + assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_DESTINATIONS is empty and the \"${case%%:*}\" fetch source is not a dump store." assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." done @@ -102,71 +153,92 @@ load ../_helper.bash popd >/dev/null } -@test "push-db: Skip for a dump store source without an upstream source" { +@test "push-db: Use the source from .env by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + stub_push_scripts + unset VORTEX_FETCH_DB_SOURCE VORTEX_PUSH_DB_DESTINATIONS export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "[ OK ] Skipped database push as VORTEX_REFRESH_DB_SOURCE is not set, so the ftp source is not a dump store." - assert_output_not_contains "Pushed to ftp." + assert_output_contains '[ OK ] Skipped database push as VORTEX_PUSH_DB_DESTINATIONS is empty and the "url" fetch source is not a dump store.' + assert_output_not_contains "Pushed to" popd >/dev/null } -@test "push-db: Use the source from .env by default" { +@test "push-db: Fail with an unsupported destination" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + stub_push_scripts - unset VORTEX_FETCH_DB_SOURCE export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_DESTINATIONS="ftp,gdrive" run .vortex/tooling/src/vortex-push-db - assert_success - assert_output_contains '[ OK ] Skipped database push as the "url" database source in VORTEX_FETCH_DB_SOURCE does not accept a push. Use one of: ftp, s3.' - assert_output_not_contains "Pushed to ftp." + assert_failure + assert_output_contains '[FAIL] Unsupported database push destination "gdrive" in VORTEX_PUSH_DB_DESTINATIONS. Use one or more of: ftp, s3.' + # Every destination is checked before the first push. + assert_output_not_contains "Pushed to" popd >/dev/null } -@test "push-db: Refuse to push a database that is not sanitized" { +@test "push-db: Refuse to push to the fetch source without an upstream source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - stub_sibling "vortex-push-db-ftp" "Pushed to ftp." + stub_push_scripts + export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_REFRESH_DB_SOURCE="" + export VORTEX_PUSH_DB_DESTINATIONS="s3,ftp" + + run .vortex/tooling/src/vortex-push-db + assert_failure + assert_output_contains '[FAIL] Refused to push the database to the "ftp" fetch source as VORTEX_REFRESH_DB_SOURCE is not set, so the push would replace the dump it fetched.' + assert_output_not_contains "Pushed to" + + popd >/dev/null +} + +@test "push-db: Refuse to push a database that is not sanitized" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_push_scripts + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_DESTINATIONS="ftp" export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 run .vortex/tooling/src/vortex-push-db assert_failure assert_output_contains "[FAIL] Refused to push the database as VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1 and the database dump is not sanitized." - assert_output_not_contains "Pushed to ftp." + assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." popd >/dev/null } -@test "push-db: Fail when the push script fails" { +@test "push-db: Stop at the first destination that fails" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 - printf '#!/usr/bin/env bash\necho "Failed to push to s3."\nexit 1\n' >.vortex/tooling/src/vortex-push-db-s3 - chmod +x .vortex/tooling/src/vortex-push-db-s3 + stub_push_scripts + printf '#!/usr/bin/env bash\necho "Failed to push to ftp."\nexit 1\n' >.vortex/tooling/src/vortex-push-db-ftp + chmod +x .vortex/tooling/src/vortex-push-db-ftp - export VORTEX_FETCH_DB_SOURCE="s3" - export VORTEX_REFRESH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_DESTINATIONS="ftp,s3" run .vortex/tooling/src/vortex-push-db assert_failure - assert_output_contains "Failed to push to s3." + assert_output_contains "Failed to push to ftp." + assert_output_not_contains "Pushed to s3" assert_output_not_contains "Finished database push." popd >/dev/null diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 9160756ad6..87ad5353cc 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -318,6 +318,33 @@ load ../_helper.bash popd >/dev/null } +@test "refresh-db: Sanitizes the database for push destinations from any fetch source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "upstream" >.data/db.sql + + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_PUSH_DB_PROCEED=1 + export VORTEX_PUSH_DB_DESTINATIONS="ftp,s3" + + stub_sibling "vortex-login-container-registry" "Logged in to the registry." + stub_sibling "vortex-export-db" 'Exported the database to ${1}.' + + mock_command "sleep" >/dev/null + mock_docker=$(mock_command "docker") + + run .vortex/tooling/src/vortex-refresh-db + assert_success + assert_output_contains "[ OK ] Finished database refresh." + + assert_string_contains "$(mock_get_call_args "${mock_docker}" 5)" "-T cli ./vendor/drevops/vortex-tooling/src/vortex-provision-sanitize-db" + assert_equal "6" "$(mock_get_call_num "${mock_docker}")" + + popd >/dev/null +} + @test "refresh-db: Exports an ftp source database unsanitized without a refresh source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 @@ -359,7 +386,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to ftp must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." + assert_output_contains "[FAIL] The pushed database dump file must be sanitized, but VORTEX_PROVISION_SANITIZE_DB_SKIP is set to 1." assert_output_not_contains "Logged in to the registry." assert_equal "0" "$(mock_get_call_num "${mock_docker}")" From 148bc9f5ed964b1a771361cae8d16c75ecc6b2ce Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 10:05:40 +1100 Subject: [PATCH 25/40] Documented the FTP directory and pushing to several destinations. --- .../content/continuous-integration/README.mdx | 4 +- .../content/development/database-storage.mdx | 48 +++++++++++++++++-- .../docs/content/development/variables.mdx | 4 ++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index dd8be65b5e..f98c5e15f9 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,8 +79,8 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry -- On the nightly run of a project with a dump store, pushes the sanitized - dump to the FTP server or S3 bucket (see [Database +- On the nightly run of a project with push destinations or a dump store, + pushes the sanitized dump to the FTP server or S3 bucket (see [Database storage](../development/database-storage.mdx#fetch-from-a-dump-store)) The job exists only when the site provisions from a database dump. With the diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 9b3102336c..6d3bbd365d 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -121,6 +121,27 @@ To make the migration source a dump store, set **Push migration DB** step pushes the migration dump as fetched, because **Vortex** sanitizes only the Drupal database. +### Push to several destinations + +The database is always fetched from 1 source, but the nightly run can push +it to several destinations. List them in `VORTEX_PUSH_DB_DESTINATIONS`, +separated by commas or spaces: + +```shell +VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 +``` + +Each destination reads its own push variables, such as +`VORTEX_PUSH_DB_FTP_HOST` or `VORTEX_PUSH_DB_S3_BUCKET`. When a destination +is also the fetch source, its push variables fall back to the fetch values. +A destination can be pushed to from any fetch source, so a project that +fetches from `lagoon` can keep sanitized copies on FTP and S3. + +When `VORTEX_PUSH_DB_DESTINATIONS` is empty, the push goes to the dump store +fetch source. The push refuses to push to the fetch source itself +without `VORTEX_REFRESH_DB_SOURCE`, because the push would replace the dump +it fetched. The migration database uses `VORTEX_PUSH_DB2_DESTINATIONS`. + ## Set up an FTP dump store Before you begin, create an FTP account that can write to a directory on @@ -135,9 +156,14 @@ do. VORTEX_FETCH_DB_SOURCE=ftp VORTEX_FETCH_DB_FTP_HOST=ftp.example.com VORTEX_FETCH_DB_FTP_PORT=21 - VORTEX_FETCH_DB_FTP_FILE=your_site/db.sql + VORTEX_FETCH_DB_FTP_DIR=your_site + VORTEX_FETCH_DB_FTP_FILE=db.sql ``` + `VORTEX_FETCH_DB_FTP_DIR` is the directory on the server, relative to + the FTP account. Leading and trailing slashes are optional. Leave it + empty for the root directory. + 2. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your upstream source, for example `lagoon`. Add the variables and secrets of that source from the table in [Dump file source for a database @@ -203,9 +229,25 @@ fails for a larger dump. On FTP, the timestamped copy is a second upload. ## Tell pushes apart +When several places push to the same store, such as GitHub Actions and +CircleCI, give each its own directory. The push directory falls back to the +fetch directory: + +| Store | Push directory | Fetch directory | +|---|---|---| +| FTP | `VORTEX_PUSH_DB_FTP_DIR` | `VORTEX_FETCH_DB_FTP_DIR` | +| S3 | `VORTEX_PUSH_DB_S3_PREFIX` | `VORTEX_FETCH_DB_S3_PREFIX` | + +For example, set `VORTEX_PUSH_DB_FTP_DIR=gha` in GitHub Actions and +`VORTEX_PUSH_DB_FTP_DIR=circleci` in CircleCI. Each provider then keeps its +own `db.sql`. A developer fetches from 1 of them with +`VORTEX_FETCH_DB_FTP_DIR` in `.env.local`. + +### Use a file suffix + `VORTEX_PUSH_DB_FILE_SUFFIX` adds a suffix before the extension of the -pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. Use a -different suffix in each place that pushes to the same store. +pushed files, such as `db_gha.sql` and `db_gha_20261111-180000.sql`. It +keeps the pushes in 1 directory. The push takes its remote filename from the fetch, then adds the suffix. To fetch a suffixed dump from an FTP store, set both filenames, so the suffix diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index c37cdc1565..752dac5909 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -320,6 +320,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_ACQUIA_DB_NAME` | Acquia database name to fetch the second database from. | `your_site` | `.env` | | `VORTEX_FETCH_DB2_ENVIRONMENT` | Environment to fetch the second database from.

Applies to hosting environments. | `prod` | `.env` | | `VORTEX_FETCH_DB2_FILE` | Second database dump file name. | `db2.sql` | `.env`, `scripts/provision-20-migration.sh` | +| `VORTEX_FETCH_DB2_FTP_DIR` | Second database dump FTP directory.

Leave empty for the root directory of the FTP account. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_FILE` | Second database dump FTP file name. | `db2.sql` | `.env` | | `VORTEX_FETCH_DB2_FTP_HOST` | Second database dump FTP host. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_FTP_PORT` | Second database dump FTP port. | `21` | `.env` | @@ -347,6 +348,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_FRESH` | Flag to fetch a fresh copy of the database by triggering a new backup. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-acquia`, `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-ftp` | +| `VORTEX_FETCH_DB_FTP_DIR` | Database dump FTP directory.

Leave empty for the root directory of the FTP account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_FILE` | Database dump FTP file name. | `db.sql` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_HOST` | Database dump FTP host. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_PASS` | Database dump FTP password. | `UNDEFINED` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db-ftp` | @@ -549,9 +551,11 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | +| `VORTEX_PUSH_DB_DESTINATIONS` | Destinations to push the database dump file to: 'ftp', 's3' or both, separated by commas or spaces.

Defaults to the fetch source when it is a dump store. Each destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for 'ftp'. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_FILE_SUFFIX` | Suffix for the names of the pushed database dump files.

Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. Use a different suffix in each place that pushes to the same storage. A fetch reads the suffixed name only when its remote file name includes it. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | +| `VORTEX_PUSH_DB_FTP_DIR` | The directory on the FTP server to push the file to.

Joined in front of the file name. Leading and trailing slashes are optional. Defaults to the directory the database is fetched from. Use a different directory in each place that pushes to the same storage, such as 'gha' and 'circleci'. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_FILE` | The remote file name, including any directories. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_HOST` | The FTP host. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_PASS` | The FTP password. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp` | From 11ac92510d626726524d64b8d9c1766f3840040b Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 10:05:47 +1100 Subject: [PATCH 26/40] Updated snapshots. --- .../Fixtures/handler_process/_baseline/.env | 11 ++++++++ .../.github/workflows/build-test-deploy.yml | 9 +++++++ .../ciprovider_circleci/.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 10 ++++++++ .../db_fetch_source_acquia/.env | 8 +++--- .../db_fetch_source_container_registry/.env | 8 +++--- .../.github/workflows/build-test-deploy.yml | 2 +- .../handler_process/db_fetch_source_ftp/.env | 25 ++++++++++--------- .../.github/workflows/build-test-deploy.yml | 16 ------------ .../db_fetch_source_lagoon/.env | 10 +++++--- .../handler_process/db_fetch_source_s3/.env | 13 +++------- .../.github/workflows/build-test-deploy.yml | 16 ------------ .../.circleci/config.yml | 10 ++++++++ .../deploy_types_all_circleci/.env | 2 +- .../handler_process/deploy_types_all_gha/.env | 2 +- .../deploy_types_artifact/.env | 2 +- .../handler_process/deploy_types_lagoon/.env | 2 +- .../.circleci/config.yml | 10 ++++++++ .../deploy_types_none_circleci/.env | 2 +- .../deploy_types_none_gha/.env | 2 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 10 ++++++++ .../handler_process/hosting_acquia/.env | 14 +++++------ .../handler_process/hosting_lagoon/.env | 14 ++++++----- .../hosting_project_name___acquia/.env | 14 +++++------ .../hosting_project_name___lagoon/.env | 14 ++++++----- .../.circleci/config.yml | 10 ++++++++ .../migration_disabled_lagoon/.env | 14 ++++++----- .../handler_process/migration_enabled/.env | 9 +++++-- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../.circleci/config.yml | 19 ++++++++++++++ .../migration_enabled_circleci/.env | 9 +++++-- .../migration_enabled_lagoon/.env | 19 +++++++++----- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../migration_fetch_source_acquia/.env | 7 +++++- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../.env | 7 +++++- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../migration_fetch_source_ftp/.env | 12 ++++++++- .../.github/workflows/build-test-deploy.yml | 8 +++--- .../migration_fetch_source_lagoon/.env | 7 +++++- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../migration_fetch_source_s3/.env | 7 +++++- .../.github/workflows/build-test-deploy.yml | 8 +++--- .../migration_fetch_source_url/.env | 9 +++++-- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../tests/Fixtures/handler_process/names/.env | 4 +-- .../notification_channels_all/.env | 4 +-- .../notification_channels_github_only/.env | 2 +- .../notification_channels_jira_only/.env | 2 +- .../notification_channels_newrelic_only/.env | 2 +- .../notification_channels_none/.env | 2 +- .../notification_channels_slack_only/.env | 2 +- .../notification_channels_webhook_only/.env | 2 +- .../provision_database_lagoon/.env | 14 ++++++----- .../handler_process/provision_profile/.env | 15 +++++++++-- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++--- .../starter_drupal_cms_profile/.env | 2 +- .../starter_drupal_profile/.env | 2 +- .../.github/workflows/build-test-deploy.yml | 4 +-- .../theme_custom_non_vortex/.env | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.github/workflows/build-test-deploy.yml | 4 +-- .../timezone_circleci/.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 6 ++--- .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 8 +++--- .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../tools_no_dclint_migration/.env | 9 +++++-- .../.github/workflows/build-test-deploy.yml | 17 ++++++++++++- .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 6 ++--- .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 10 ++++++++ .../.github/workflows/build-test-deploy.yml | 10 ++++---- .../handler_process/version_scheme_other/.env | 2 +- .../version_scheme_semver/.env | 2 +- 101 files changed, 640 insertions(+), 215 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env index fe35eea839..bc6b408606 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env @@ -187,6 +187,17 @@ VORTEX_FETCH_DB_SOURCE=url # HTTP Basic Authentication credentials should be embedded into the value. VORTEX_FETCH_DB_URL= +# Destinations the nightly CI job pushes the sanitized database dump to: +# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source +# when it is a dump store. Each destination reads its own variables, such as +# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 +# source. +# VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 + +# Also push a copy of each dump named with a UTC timestamp, such as +# db_20261111-180000.sql, to keep a history of the pushed dumps. +# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + # Environment to fetch the database from. # # Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index 3580a1e345..cf49a27441 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -278,6 +278,15 @@ jobs: ./vendor/bin/vortex-refresh-db timeout-minutes: 30 + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - name: Push DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order diff --git a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml index 613d58ed05..fc8bc445cb 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -492,6 +492,17 @@ +@@ -501,6 +501,17 @@ hide_and_recreate: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml index ea65c0eaad..46bc8293e1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env index 87364d8797..9cdae39af9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env @@ -10,10 +10,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +249,3 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -265,17 +260,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index f98032044d..c9b8a0278f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -19,10 +19,10 @@ +# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +259,3 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -265,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 3086561937..748874cfc7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -24,4 +24,4 @@ + ./vendor/bin/vortex-push-db-image timeout-minutes: 30 - # Save cache per default branch and the timestamp. + # Push the freshly-exported sanitized database dump to the push diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index 8d266c9412..36f8efc921 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,32 @@ +@@ -180,13 +180,33 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -14,28 +14,29 @@ +# Database dump FTP port. +VORTEX_FETCH_DB_FTP_PORT=21 + ++# Database dump FTP directory. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Leave empty for the root directory of the FTP account. ++VORTEX_FETCH_DB_FTP_DIR= + +# Database dump FTP file name. +VORTEX_FETCH_DB_FTP_FILE=db.sql + +# Upstream source of the database dump file, such as url, acquia or lagoon. - # --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= ++# +# Uncomment to make the database fetch source a dump store: the nightly CI +# job fetches the dump from this source, sanitizes it and pushes it to the +# fetch source as the latest dump. The fetch script of this source reads its +# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep +# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url - -+# Also push a copy of each dump named with a UTC timestamp, such as -+# db_20261111-180000.sql, to keep a history of the pushed dumps. -+# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 + - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +273,3 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -265,17 +285,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml index 99ebadce06..6ed86e851c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -7,19 +7,3 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -276,6 +278,15 @@ - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi - ./vendor/bin/vortex-refresh-db -+ timeout-minutes: 30 -+ -+ # Push the freshly-exported sanitized database dump to the database fetch -+ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the -+ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). -+ - name: Push DB -+ run: | -+ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi -+ ./vendor/bin/vortex-push-db - timeout-minutes: 30 - - # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env index c977899158..9fba49ca12 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env @@ -1,4 +1,4 @@ -@@ -180,19 +180,14 @@ +@@ -180,13 +180,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -10,8 +10,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,7 +198,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -20,7 +22,7 @@ ################################################################################ # FILES SOURCE # -@@ -254,17 +249,3 @@ +@@ -265,17 +260,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index aa442ec9c3..591f9ffe6d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,29 @@ +@@ -180,12 +180,24 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -25,14 +25,9 @@ +# secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url -+# Also push a copy of each dump named with a UTC timestamp, such as -+# db_20261111-180000.sql, to keep a history of the pushed dumps. -+# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 -+ - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -254,17 +270,3 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source +@@ -265,17 +277,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml index b95ee1f3c8..a9c60fbab3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -7,19 +7,3 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -276,6 +278,15 @@ - run: | - if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi - ./vendor/bin/vortex-refresh-db -+ timeout-minutes: 30 -+ -+ # Push the freshly-exported sanitized database dump to the database fetch -+ # source. No-op unless a fresh DB was fetched (the fetch semaphore) and the -+ # push was requested (VORTEX_PUSH_DB_PROCEED, in the script). -+ - name: Push DB -+ run: | -+ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi -+ ./vendor/bin/vortex-push-db - timeout-minutes: 30 - - # Save cache per default branch and the timestamp. diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env index cd44afc636..fff488a98d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env @@ -1,4 +1,4 @@ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env index cd44afc636..fff488a98d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env @@ -1,4 +1,4 @@ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env index 43c6e69380..aec0c46a96 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env @@ -1,4 +1,4 @@ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env index 603024b6e0..65058d435a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env @@ -1,4 +1,4 @@ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml index 6b2526e489..ed97a2c0fa 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env index f77a701c0b..64c4894783 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env @@ -1,4 +1,4 @@ -@@ -222,14 +222,6 @@ +@@ -233,14 +233,6 @@ VORTEX_RELEASE_VERSION_SCHEME=calver ################################################################################ diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env index f77a701c0b..64c4894783 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env @@ -1,4 +1,4 @@ -@@ -222,14 +222,6 @@ +@@ -233,14 +233,6 @@ VORTEX_RELEASE_VERSION_SCHEME=calver ################################################################################ diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index d0b2146bf0..7b5594d096 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -575,104 +575,3 @@ +@@ -584,104 +584,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env index 77b53e0608..9cc68201a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env @@ -33,10 +33,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -194,6 +196,9 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -205,6 +207,9 @@ # a branch name or an environment name. VORTEX_FETCH_DB_ENVIRONMENT=prod @@ -46,7 +46,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -202,15 +207,8 @@ +@@ -213,15 +218,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -63,7 +63,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +225,7 @@ +@@ -238,7 +236,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -72,7 +72,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +252,3 @@ +@@ -265,17 +263,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env index 626f08b424..a1d35bca50 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env @@ -17,7 +17,7 @@ # DATABASE SOURCE # ################################################################################ -@@ -180,19 +192,14 @@ +@@ -180,13 +192,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -29,8 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,7 +210,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -39,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -202,15 +209,8 @@ +@@ -213,15 +220,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -56,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -65,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env index 0da09ab1f2..65467f110a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env @@ -33,10 +33,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # - # Applies to hosting environments. -@@ -194,6 +196,9 @@ + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -205,6 +207,9 @@ # a branch name or an environment name. VORTEX_FETCH_DB_ENVIRONMENT=prod @@ -46,7 +46,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -202,15 +207,8 @@ +@@ -213,15 +218,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -63,7 +63,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +225,7 @@ +@@ -238,7 +236,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -72,7 +72,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +252,3 @@ +@@ -265,17 +263,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env index 009b57271b..0689bf7b1b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env @@ -17,7 +17,7 @@ # DATABASE SOURCE # ################################################################################ -@@ -180,19 +192,14 @@ +@@ -180,13 +192,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -29,8 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,7 +210,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -39,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -202,15 +209,8 @@ +@@ -213,15 +220,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -56,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -65,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env index 626f08b424..a1d35bca50 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env @@ -17,7 +17,7 @@ # DATABASE SOURCE # ################################################################################ -@@ -180,19 +192,14 @@ +@@ -180,13 +192,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -29,8 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,7 +210,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -39,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -202,15 +209,8 @@ +@@ -213,15 +220,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -56,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -65,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env index 93beae49d2..ac0a01196b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,29 @@ +@@ -206,6 +206,34 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,6 +19,11 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. @@ -28,7 +33,7 @@ # FILES SOURCE # ################################################################################ -@@ -268,3 +291,6 @@ +@@ -279,3 +307,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml index 460b6bc35f..35ffec4ffe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml @@ -8,7 +8,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +436,11 @@ +@@ -287,6 +290,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +453,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml index 657e41c70f..49de9d5ef1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml @@ -294,6 +294,25 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + + # Push the freshly-fetched migration database dump to its push destinations, + # next to the primary dump. No-op unless the push was requested. + - run: + name: Push migration DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env index 93beae49d2..ac0a01196b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,29 @@ +@@ -206,6 +206,34 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,6 +19,11 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. @@ -28,7 +33,7 @@ # FILES SOURCE # ################################################################################ -@@ -268,3 +291,6 @@ +@@ -279,3 +307,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env index b8b0fbc8fd..005f07c4a4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env @@ -17,7 +17,7 @@ # DATABASE SOURCE # ################################################################################ -@@ -180,21 +192,34 @@ +@@ -180,13 +192,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -29,8 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,9 +210,32 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -50,6 +52,11 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=lagoon + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. @@ -59,7 +66,7 @@ # FILES SOURCE # ################################################################################ -@@ -202,15 +227,8 @@ +@@ -213,15 +243,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -76,7 +83,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +245,7 @@ +@@ -238,7 +261,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -85,7 +92,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +272,3 @@ +@@ -265,17 +288,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml index 460b6bc35f..35ffec4ffe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml @@ -8,7 +8,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +436,11 @@ +@@ -287,6 +290,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +453,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env index 9c85db4822..30942115aa 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,24 @@ +@@ -206,6 +206,29 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -14,6 +14,11 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=acquia + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml index 460b6bc35f..35ffec4ffe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -8,7 +8,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +436,11 @@ +@@ -287,6 +290,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +453,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index cdfd843852..ae32230e6f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,39 @@ +@@ -206,6 +206,44 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -29,6 +29,11 @@ +# in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index e819850e04..38db031910 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -23,7 +23,22 @@ ./vendor/bin/vortex-refresh-db timeout-minutes: 30 -@@ -433,6 +441,11 @@ +@@ -287,6 +295,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env index ae7d4bb2fe..aaa0ad57b2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,42 @@ +@@ -206,6 +206,52 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -22,6 +22,11 @@ +# Second database dump FTP port. +VORTEX_FETCH_DB2_FTP_PORT=21 + ++# Second database dump FTP directory. ++# ++# Leave empty for the root directory of the FTP account. ++VORTEX_FETCH_DB2_FTP_DIR= ++ +# Second database dump FTP file name. +VORTEX_FETCH_DB2_FTP_FILE=db2.sql + @@ -32,6 +37,11 @@ +# to the fetch source, next to the primary dump. +# VORTEX_REFRESH_DB2_SOURCE=url + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml index bf29fc7cbc..e4582dfc2a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -17,11 +17,11 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -278,6 +283,14 @@ - ./vendor/bin/vortex-refresh-db +@@ -287,6 +292,14 @@ + ./vendor/bin/vortex-push-db timeout-minutes: 30 -+ # Push the freshly-fetched migration database dump to its fetch source, ++ # Push the freshly-fetched migration database dump to its push destinations, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | @@ -32,7 +32,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -433,6 +446,11 @@ +@@ -442,6 +455,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env index 525b75b5aa..ddd07958cb 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,24 @@ +@@ -206,6 +206,29 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -14,6 +14,11 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=lagoon + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml index 460b6bc35f..35ffec4ffe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml @@ -8,7 +8,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +436,11 @@ +@@ -287,6 +290,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +453,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env index b1aec4e0f5..129946b761 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,39 @@ +@@ -206,6 +206,44 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -29,6 +29,11 @@ +# to the fetch source, next to the primary dump. +# VORTEX_REFRESH_DB2_SOURCE=url + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml index 283b4f1f72..436837f2c7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -17,11 +17,11 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -278,6 +283,14 @@ - ./vendor/bin/vortex-refresh-db +@@ -287,6 +292,14 @@ + ./vendor/bin/vortex-push-db timeout-minutes: 30 -+ # Push the freshly-fetched migration database dump to its fetch source, ++ # Push the freshly-fetched migration database dump to its push destinations, + # next to the primary dump. No-op unless the push was requested. + - name: Push migration DB + run: | @@ -32,7 +32,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -433,6 +446,11 @@ +@@ -442,6 +455,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env index 93beae49d2..ac0a01196b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,29 @@ +@@ -206,6 +206,34 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,6 +19,11 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. @@ -28,7 +33,7 @@ # FILES SOURCE # ################################################################################ -@@ -268,3 +291,6 @@ +@@ -279,3 +307,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml index 460b6bc35f..35ffec4ffe 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml @@ -8,7 +8,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +436,11 @@ +@@ -287,6 +290,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +453,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml index ab7610ac1b..210025dfb1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -314,7 +314,6 @@ +@@ -323,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +500,6 @@ +@@ -510,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/names/.env b/.vortex/installer/tests/Fixtures/handler_process/names/.env index c268d92b0e..907f094118 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/names/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/names/.env @@ -42,7 +42,7 @@ # Put the site into a maintenance mode during site provisioning. VORTEX_PROVISION_USE_MAINTENANCE_MODE=1 -@@ -244,7 +244,7 @@ +@@ -255,7 +255,7 @@ # An email address to send notifications from. # # Applies to email notifications. @@ -51,7 +51,7 @@ # Email address(es) to send notifications to. # -@@ -253,7 +253,7 @@ +@@ -264,7 +264,7 @@ # Multiple names can be specified as a comma-separated list of email addresses # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env index e0bcbf0605..2cdcb2a1e6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env @@ -1,4 +1,4 @@ -@@ -239,7 +239,7 @@ +@@ -250,7 +250,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy @@ -7,7 +7,7 @@ # An email address to send notifications from. # -@@ -254,6 +254,12 @@ +@@ -265,6 +265,12 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env index 485b37b7d6..1a30a0585a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,7 @@ +@@ -250,21 +250,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env index 9d42fdec49..0491e177eb 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,10 @@ +@@ -250,21 +250,10 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env index a8436d62ad..2b5368dd47 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,7 @@ +@@ -250,21 +250,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env index f16700058d..1338f61c23 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,7 @@ +@@ -250,21 +250,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env index d5ee87a619..bfda460b3e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,7 @@ +@@ -250,21 +250,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env index 33396c5921..e2de45c9a0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env @@ -1,4 +1,4 @@ -@@ -239,21 +239,10 @@ +@@ -250,21 +250,10 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env index 626f08b424..a1d35bca50 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env @@ -17,7 +17,7 @@ # DATABASE SOURCE # ################################################################################ -@@ -180,19 +192,14 @@ +@@ -180,13 +192,8 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -29,8 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # when it is a dump store. Each destination reads its own variables, such as +@@ -203,7 +210,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -39,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -202,15 +209,8 @@ +@@ -213,15 +220,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -56,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -227,7 +227,7 @@ +@@ -238,7 +238,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -65,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env index 377af86429..e86cf6c82d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env @@ -7,7 +7,7 @@ # Overwrite a database if it exists. # -@@ -179,21 +179,6 @@ +@@ -179,32 +179,6 @@ # The file is used to import the database into an empty database container. VORTEX_DB_FILE=db.sql @@ -19,6 +19,17 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - +-# Destinations the nightly CI job pushes the sanitized database dump to: +-# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source +-# when it is a dump store. Each destination reads its own variables, such as +-# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 +-# source. +-# VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 +- +-# Also push a copy of each dump named with a UTC timestamp, such as +-# db_20261111-180000.sql, to keep a history of the pushed dumps. +-# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 +- -# Environment to fetch the database from. -# -# Applies to hosting environments. @@ -29,7 +40,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -254,17 +239,3 @@ +@@ -265,17 +239,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index 0dfba85dcf..f54fa2939e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -15,7 +15,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -167,132 +164,8 @@ +@@ -167,141 +164,8 @@ run: docker compose exec -T cli bash -c "npm --prefix=\${WEBROOT}/themes/custom/\${DRUPAL_THEME} run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -130,6 +130,15 @@ - ./vendor/bin/vortex-refresh-db - timeout-minutes: 30 - +- # Push the freshly-exported sanitized database dump to the push +- # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and +- # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). +- - name: Push DB +- run: | +- if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi +- ./vendor/bin/vortex-push-db +- timeout-minutes: 30 +- - # Save cache per default branch and the timestamp. - # The cache will not be saved if it already exists. - # Note that the cache fallback flag is enabled for this case in order @@ -148,7 +157,7 @@ permissions: contents: read -@@ -335,14 +208,6 @@ +@@ -344,14 +208,6 @@ VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING: "1" VORTEX_SSH_REMOVE_ALL_KEYS: "1" VORTEX_DEBUG: ${{ vars.VORTEX_DEBUG }} @@ -163,7 +172,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -381,29 +246,6 @@ +@@ -390,29 +246,6 @@ - name: Install Vortex tooling run: ./scripts/vortex-tooling.sh @@ -193,7 +202,7 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -579,7 +421,6 @@ +@@ -588,7 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] diff --git a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env index 5666759cb8..26fc5387c9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env @@ -7,7 +7,7 @@ # Drupal configuration directory. # -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env index f745b08688..e10eafb327 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env @@ -1,4 +1,4 @@ -@@ -254,17 +254,3 @@ +@@ -265,17 +265,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml index 0cd1802313..fb4ec49edc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -314,7 +309,6 @@ +@@ -323,7 +318,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +495,6 @@ +@@ -510,18 +504,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env b/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env index b98e7c9da3..5a3382abdd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env @@ -5,5 +5,5 @@ -VORTEX_FETCH_DB_URL= +VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql - # Environment to fetch the database from. - # + # Destinations the nightly CI job pushes the sanitized database dump to: + # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml index 0cd1802313..fb4ec49edc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -314,7 +309,6 @@ +@@ -323,7 +318,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +495,6 @@ +@@ -510,18 +504,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml index 0cd1802313..fb4ec49edc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -314,7 +309,6 @@ +@@ -323,7 +318,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +495,6 @@ +@@ -510,18 +504,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml index 56e766a652..d8f25d774d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml @@ -277,6 +277,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml index e5104a12f6..b71d452b3f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -313,14 +309,7 @@ +@@ -322,14 +318,7 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -24,7 +24,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -442,66 +431,6 @@ +@@ -451,66 +440,6 @@ run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} @@ -91,7 +91,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -514,19 +443,6 @@ +@@ -523,19 +452,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -111,7 +111,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -543,16 +459,6 @@ +@@ -552,16 +468,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml index 4125037c62..bc5ca2b53f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml index 4d1ca474b4..8116ff4305 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,7 +302,6 @@ +@@ -321,7 +311,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -30,7 +30,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -423,7 +412,6 @@ +@@ -432,7 +421,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -38,7 +38,7 @@ - name: Provision site run: | -@@ -436,11 +424,6 @@ +@@ -445,11 +433,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml index d21bef58a2..ff41066688 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml @@ -268,6 +268,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml index 3245313f2f..d10899d1e4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,9 +302,7 @@ +@@ -321,9 +311,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -32,7 +32,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -423,7 +411,6 @@ +@@ -432,7 +420,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -40,7 +40,7 @@ - name: Provision site run: | -@@ -437,11 +424,6 @@ +@@ -446,11 +433,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -52,7 +52,7 @@ - name: Test with PHPUnit if: ${{ env.VORTEX_CI_IS_PHPUNIT_RUNNER == 'true' }} run: docker compose exec -T cli vendor/bin/phpunit -@@ -501,18 +483,6 @@ +@@ -510,18 +492,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml index 8e015f51d7..d5693d7f8b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml @@ -259,6 +259,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml index fd50aad45b..d12b0c686a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -315,12 +311,6 @@ +@@ -324,12 +320,6 @@ VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -22,7 +22,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -513,19 +503,6 @@ +@@ -522,19 +512,6 @@ exit 1 fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml index 4735766d37..090353bbea 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml index 54dc4477e3..96c7a95e27 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml @@ -280,6 +280,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env index 93beae49d2..ac0a01196b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env @@ -1,4 +1,4 @@ -@@ -195,6 +195,29 @@ +@@ -206,6 +206,34 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,6 +19,11 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched: ftp, s3 or both. Defaults to the second database fetch source when ++# it is a dump store. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ +# Environment to fetch the second database from. +# +# Applies to hosting environments. @@ -28,7 +33,7 @@ # FILES SOURCE # ################################################################################ -@@ -268,3 +291,6 @@ +@@ -279,3 +307,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml index 03afc8216b..5c873888ef 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml @@ -19,7 +19,22 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -433,6 +432,11 @@ +@@ -287,6 +286,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -442,6 +449,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml index fffe322f5d..0aa494e797 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml @@ -273,6 +273,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml index 0cd1802313..fb4ec49edc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -314,7 +309,6 @@ +@@ -323,7 +318,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +495,6 @@ +@@ -510,18 +504,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml index 121b9f5477..a0c33f49b3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml @@ -282,6 +282,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml index 2963f464ca..d2ac4714a0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -312,7 +312,6 @@ +@@ -321,7 +321,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -423,7 +422,6 @@ +@@ -432,7 +431,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -14,7 +14,7 @@ - name: Provision site run: | -@@ -436,11 +434,6 @@ +@@ -445,11 +443,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml index 8cbd50d31b..6344469422 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml index 3262246832..1d990b3080 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml index 8fc685f729..06fff5ff4e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml index 4bf951f78a..0459fb8811 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -313,7 +313,6 @@ +@@ -322,7 +322,6 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true -@@ -441,66 +440,6 @@ +@@ -450,66 +449,6 @@ if: ${{ env.VORTEX_CI_IS_JEST_RUNNER == 'true' }} run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml index 05b7fbe608..95607bec4c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml index dfdbda3a2a..61b6f2744f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml index b59dde79d2..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml index 0cd1802313..fb4ec49edc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -314,7 +309,6 @@ +@@ -323,7 +318,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -501,18 +495,6 @@ +@@ -510,18 +504,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml index f2890ba107..4b18ee9a34 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml @@ -285,6 +285,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml index e3c358bb75..864bb33983 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml @@ -58,7 +58,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -312,15 +271,7 @@ +@@ -321,15 +280,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -74,7 +74,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -423,7 +374,6 @@ +@@ -432,7 +383,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -82,7 +82,7 @@ - name: Provision site run: | -@@ -437,71 +387,6 @@ +@@ -446,71 +396,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -154,7 +154,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -514,19 +399,6 @@ +@@ -523,19 +408,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -174,7 +174,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -543,16 +415,6 @@ +@@ -552,16 +424,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env index 7217f94ddf..fd79e009be 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env @@ -1,4 +1,4 @@ -@@ -219,7 +219,7 @@ +@@ -230,7 +230,7 @@ # # Can be one of: calver, semver, other # @see https://www.vortextemplate.com/docs/releasing diff --git a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env index 4bf3e13c95..e01d4eb267 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env @@ -1,4 +1,4 @@ -@@ -219,7 +219,7 @@ +@@ -230,7 +230,7 @@ # # Can be one of: calver, semver, other # @see https://www.vortextemplate.com/docs/releasing From 4cf30a2482755d303d88dac72b8aba1155bd2a60 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 10:09:32 +1100 Subject: [PATCH 27/40] Reworded the timestamped copy note in '.env' so the configured timezone stays the only one named. --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 2ca87bb620..9356e39918 100644 --- a/.env +++ b/.env @@ -281,7 +281,7 @@ VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 # source. # VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 -# Also push a copy of each dump named with a UTC timestamp, such as +# Also push a copy of each dump named with the time of the push, such as # db_20261111-180000.sql, to keep a history of the pushed dumps. # VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 From 305f54d99ab996dfbcead241004cf1dd4bc9f07b Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Thu, 12 Nov 2026 10:09:39 +1100 Subject: [PATCH 28/40] Updated snapshots. --- .vortex/installer/tests/Fixtures/handler_process/_baseline/.env | 2 +- .../tests/Fixtures/handler_process/provision_profile/.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env index bc6b408606..8a3c374718 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env @@ -194,7 +194,7 @@ VORTEX_FETCH_DB_URL= # source. # VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 -# Also push a copy of each dump named with a UTC timestamp, such as +# Also push a copy of each dump named with the time of the push, such as # db_20261111-180000.sql, to keep a history of the pushed dumps. # VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env index e86cf6c82d..3def98864d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env @@ -26,7 +26,7 @@ -# source. -# VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 - --# Also push a copy of each dump named with a UTC timestamp, such as +-# Also push a copy of each dump named with the time of the push, such as -# db_20261111-180000.sql, to keep a history of the pushed dumps. -# VORTEX_PUSH_DB_TIMESTAMPED_COPY=1 - From 2de43555be2f3a3f5a2c1d4615d5585182d8967b Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 09:21:04 +1100 Subject: [PATCH 29/40] Pushed Vortex's own demo databases to the destinations in the 'VORTEX_PUSH_DB_DESTINATIONS' and 'VORTEX_PUSH_DB2_DESTINATIONS' variables, and fell back to the primary push server for an indexed database. --- .circleci/config.yml | 8 ++-- .circleci/vortex-test-common.yml | 3 -- .github/workflows/build-test-deploy.yml | 13 +++--- .github/workflows/vortex-test-didi.yml | 26 ++++++------ .../content/development/database-storage.mdx | 8 +++- .vortex/tooling/src/vortex-push-db-ftp | 13 +++--- .vortex/tooling/src/vortex-push-db-s3 | 13 +++--- .vortex/tooling/tests/unit/push-db-ftp.bats | 42 +++++++++++++++++++ .vortex/tooling/tests/unit/push-db-s3.bats | 40 ++++++++++++++++++ 9 files changed, 128 insertions(+), 38 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26836dc8d7..8ec242ba7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -371,9 +371,6 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db - #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi - #;> VORTEX_DEV no_output_timeout: 30m #;< MIGRATION @@ -415,7 +412,10 @@ jobs: VORTEX_PUSH_DB_PROCEED: 1 #;< VORTEX_DEV # Tell this provider's demo database dumps apart on the test FTP server - # by directory, with a timestamped copy of each push. + # by directory, with a timestamped copy of each push. The destinations + # and the FTP server come from the VORTEX_PUSH_DB_DESTINATIONS, + # VORTEX_PUSH_DB2_DESTINATIONS and VORTEX_PUSH_DB_FTP_* project + # variables. VORTEX_PUSH_DB_FTP_DIR: circleci VORTEX_PUSH_DB_TIMESTAMPED_COPY: 1 #;> VORTEX_DEV diff --git a/.circleci/vortex-test-common.yml b/.circleci/vortex-test-common.yml index 36d2f77968..34dde7ea56 100644 --- a/.circleci/vortex-test-common.yml +++ b/.circleci/vortex-test-common.yml @@ -191,9 +191,6 @@ jobs: command: | [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 ./vendor/bin/vortex-push-db - #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi - #;> VORTEX_DEV no_output_timeout: 30m #;< MIGRATION diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index c3c0f7b770..a74d249eae 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -242,9 +242,13 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, in a - # directory that tells this provider's dumps apart, with a timestamped - # copy of each push. + # Push destinations of the demo databases on the nightly run. Empty + # repository variables skip the push. + VORTEX_PUSH_DB_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB_DESTINATIONS }} + VORTEX_PUSH_DB2_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB2_DESTINATIONS }} + # FTP server that receives both demo databases, in a directory that + # tells this provider's dumps apart, with a timestamped copy of each + # push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} @@ -380,9 +384,6 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db - #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi - #;> VORTEX_DEV timeout-minutes: 30 #;< MIGRATION diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index e5b891a370..af04199f88 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -81,9 +81,13 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, in a - # directory that tells this provider's dumps apart, with a timestamped - # copy of each push. + # Push destinations of the demo databases on the nightly run. Empty + # repository variables skip the push. + VORTEX_PUSH_DB_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB_DESTINATIONS }} + VORTEX_PUSH_DB2_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB2_DESTINATIONS }} + # FTP server that receives both demo databases, in a directory that + # tells this provider's dumps apart, with a timestamped copy of each + # push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} @@ -228,9 +232,6 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db - #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi - #;> VORTEX_DEV timeout-minutes: 30 #;< MIGRATION @@ -655,9 +656,13 @@ jobs: VORTEX_FETCH_DB2_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB2_S3_SECRET_KEY }} #;> MIGRATION_DB_FETCH_SOURCE_S3 #;< VORTEX_DEV - # FTP server that receives the demo database on the nightly run, in a - # directory that tells this provider's dumps apart, with a timestamped - # copy of each push. + # Push destinations of the demo databases on the nightly run. Empty + # repository variables skip the push. + VORTEX_PUSH_DB_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB_DESTINATIONS }} + VORTEX_PUSH_DB2_DESTINATIONS: ${{ vars.VORTEX_PUSH_DB2_DESTINATIONS }} + # FTP server that receives both demo databases, in a directory that + # tells this provider's dumps apart, with a timestamped copy of each + # push. VORTEX_PUSH_DB_FTP_HOST: ${{ secrets.VORTEX_PUSH_DB_FTP_HOST }} VORTEX_PUSH_DB_FTP_PORT: ${{ secrets.VORTEX_PUSH_DB_FTP_PORT }} VORTEX_PUSH_DB_FTP_USER: ${{ secrets.VORTEX_PUSH_DB_FTP_USER }} @@ -820,9 +825,6 @@ jobs: run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ./vendor/bin/vortex-push-db - #;< VORTEX_DEV - if [ -n "${VORTEX_PUSH_DB_FTP_HOST:-}" ]; then VORTEX_PUSH_DB_DESTINATIONS=ftp ./vendor/bin/vortex-push-db; fi - #;> VORTEX_DEV timeout-minutes: 30 #;< MIGRATION diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 6d3bbd365d..839cbaf496 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -140,7 +140,13 @@ fetches from `lagoon` can keep sanitized copies on FTP and S3. When `VORTEX_PUSH_DB_DESTINATIONS` is empty, the push goes to the dump store fetch source. The push refuses to push to the fetch source itself without `VORTEX_REFRESH_DB_SOURCE`, because the push would replace the dump -it fetched. The migration database uses `VORTEX_PUSH_DB2_DESTINATIONS`. +it fetched. + +The migration database reads its own list in +`VORTEX_PUSH_DB2_DESTINATIONS`. Its connection values, such as +`VORTEX_PUSH_DB2_FTP_HOST`, fall back to the primary push values, so both +databases can share 1 server or bucket. Each database keeps its own file +name. ## Set up an FTP dump store diff --git a/.vortex/tooling/src/vortex-push-db-ftp b/.vortex/tooling/src/vortex-push-db-ftp index 0cfa7ce2ed..06529c958e 100755 --- a/.vortex/tooling/src/vortex-push-db-ftp +++ b/.vortex/tooling/src/vortex-push-db-ftp @@ -24,27 +24,28 @@ set -eu _db_index="${VORTEX_DB_INDEX:-}" # The FTP values fall back to the values the database is fetched with, so a -# server used as the fetch source is configured once. +# server used as the fetch source is configured once. An indexed database then +# falls back to the primary push values, so several databases share 1 server. # The FTP user. _v="VORTEX_PUSH_DB${_db_index}_FTP_USER" _vs="VORTEX_FETCH_DB${_db_index}_FTP_USER" -VORTEX_PUSH_DB_FTP_USER="${!_v:-${!_vs:-}}" +VORTEX_PUSH_DB_FTP_USER="${!_v:-${!_vs:-${VORTEX_PUSH_DB_FTP_USER:-}}}" # The FTP password. _v="VORTEX_PUSH_DB${_db_index}_FTP_PASS" _vs="VORTEX_FETCH_DB${_db_index}_FTP_PASS" -VORTEX_PUSH_DB_FTP_PASS="${!_v:-${!_vs:-}}" +VORTEX_PUSH_DB_FTP_PASS="${!_v:-${!_vs:-${VORTEX_PUSH_DB_FTP_PASS:-}}}" # The FTP host. _v="VORTEX_PUSH_DB${_db_index}_FTP_HOST" _vs="VORTEX_FETCH_DB${_db_index}_FTP_HOST" -VORTEX_PUSH_DB_FTP_HOST="${!_v:-${!_vs:-}}" +VORTEX_PUSH_DB_FTP_HOST="${!_v:-${!_vs:-${VORTEX_PUSH_DB_FTP_HOST:-}}}" # The FTP port. _v="VORTEX_PUSH_DB${_db_index}_FTP_PORT" _vs="VORTEX_FETCH_DB${_db_index}_FTP_PORT" -VORTEX_PUSH_DB_FTP_PORT="${!_v:-${!_vs:-21}}" +VORTEX_PUSH_DB_FTP_PORT="${!_v:-${!_vs:-${VORTEX_PUSH_DB_FTP_PORT:-21}}}" # The remote file name, including any directories. _v="VORTEX_PUSH_DB${_db_index}_FTP_FILE" @@ -59,7 +60,7 @@ VORTEX_PUSH_DB_FTP_FILE="${!_v:-${!_vs:-}}" # 'circleci'. _v="VORTEX_PUSH_DB${_db_index}_FTP_DIR" _vs="VORTEX_FETCH_DB${_db_index}_FTP_DIR" -VORTEX_PUSH_DB_FTP_DIR="${!_v:-${!_vs:-}}" +VORTEX_PUSH_DB_FTP_DIR="${!_v:-${!_vs:-${VORTEX_PUSH_DB_FTP_DIR:-}}}" # Directory with database dump file. _v="VORTEX_PUSH_DB${_db_index}_FTP_DB_DIR" diff --git a/.vortex/tooling/src/vortex-push-db-s3 b/.vortex/tooling/src/vortex-push-db-s3 index 0a9ce5ea95..d7583a060a 100755 --- a/.vortex/tooling/src/vortex-push-db-s3 +++ b/.vortex/tooling/src/vortex-push-db-s3 @@ -24,32 +24,33 @@ set -o pipefail _db_index="${VORTEX_DB_INDEX:-}" # The S3 values fall back to the values the database is fetched with, so a -# bucket used as the fetch source is configured once. +# bucket used as the fetch source is configured once. An indexed database then +# falls back to the primary push values, so several databases share 1 bucket. # AWS access key. _v="VORTEX_PUSH_DB${_db_index}_S3_ACCESS_KEY" _vs="VORTEX_FETCH_DB${_db_index}_S3_ACCESS_KEY" -VORTEX_PUSH_DB_S3_ACCESS_KEY="${!_v:-${!_vs:-${S3_ACCESS_KEY:-}}}" +VORTEX_PUSH_DB_S3_ACCESS_KEY="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_ACCESS_KEY:-${S3_ACCESS_KEY:-}}}}" # AWS secret key. _v="VORTEX_PUSH_DB${_db_index}_S3_SECRET_KEY" _vs="VORTEX_FETCH_DB${_db_index}_S3_SECRET_KEY" -VORTEX_PUSH_DB_S3_SECRET_KEY="${!_v:-${!_vs:-${S3_SECRET_KEY:-}}}" +VORTEX_PUSH_DB_S3_SECRET_KEY="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_SECRET_KEY:-${S3_SECRET_KEY:-}}}}" # S3 bucket name. _v="VORTEX_PUSH_DB${_db_index}_S3_BUCKET" _vs="VORTEX_FETCH_DB${_db_index}_S3_BUCKET" -VORTEX_PUSH_DB_S3_BUCKET="${!_v:-${!_vs:-${S3_BUCKET:-}}}" +VORTEX_PUSH_DB_S3_BUCKET="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_BUCKET:-${S3_BUCKET:-}}}}" # S3 region. _v="VORTEX_PUSH_DB${_db_index}_S3_REGION" _vs="VORTEX_FETCH_DB${_db_index}_S3_REGION" -VORTEX_PUSH_DB_S3_REGION="${!_v:-${!_vs:-${S3_REGION:-}}}" +VORTEX_PUSH_DB_S3_REGION="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_REGION:-${S3_REGION:-}}}}" # S3 prefix (path within the bucket). _v="VORTEX_PUSH_DB${_db_index}_S3_PREFIX" _vs="VORTEX_FETCH_DB${_db_index}_S3_PREFIX" -VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${S3_PREFIX:-}}}" +VORTEX_PUSH_DB_S3_PREFIX="${!_v:-${!_vs:-${VORTEX_PUSH_DB_S3_PREFIX:-${S3_PREFIX:-}}}}" # Directory with database dump file. _v="VORTEX_PUSH_DB${_db_index}_S3_DB_DIR" diff --git a/.vortex/tooling/tests/unit/push-db-ftp.bats b/.vortex/tooling/tests/unit/push-db-ftp.bats index bf138d08a9..a13d8745a1 100644 --- a/.vortex/tooling/tests/unit/push-db-ftp.bats +++ b/.vortex/tooling/tests/unit/push-db-ftp.bats @@ -151,6 +151,48 @@ setup_ftp_fetch_values() { popd >/dev/null } +@test "push-db-ftp: Push an indexed database to the primary push server" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + setup_ftp_fetch_values + export VORTEX_PUSH_DB_FTP_USER="push-user" + export VORTEX_PUSH_DB_FTP_PASS="push-pass" + export VORTEX_PUSH_DB_FTP_HOST="push.example.com" + export VORTEX_PUSH_DB_FTP_PORT="2121" + export VORTEX_PUSH_DB_FTP_DIR="gha" + export VORTEX_PUSH_DB_FTP_FILE="db.sql" + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_FTP_USER="" + export VORTEX_FETCH_DB2_FTP_PASS="" + export VORTEX_FETCH_DB2_FTP_HOST="" + export VORTEX_FETCH_DB2_FTP_PORT="" + export VORTEX_FETCH_DB2_FTP_DIR="" + export VORTEX_FETCH_DB2_FTP_FILE="db2.sql" + export VORTEX_FETCH_DB2_FILE="db2.sql" + + mock_curl=$(mock_command "curl") + mock_set_side_effect "${mock_curl}" "cat >${BUILD_DIR}/curl-config" 1 + + run .vortex/tooling/src/vortex-push-db-ftp + assert_success + # The server and the directory are shared, while the file stays the + # indexed database's own. + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: gha/db2.sql" + assert_output_contains "FTP host: push.example.com:2121" + assert_output_contains "[ OK ] Finished database 2 dump push to FTP." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "ftp://push.example.com:2121/gha/db2.sql.uploading" + assert_file_contains "${BUILD_DIR}/curl-config" 'user = "push-user:push-pass"' + + rm -rf .data + + popd >/dev/null +} + @test "push-db-ftp: Escape quotes and backslashes in the credentials" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 diff --git a/.vortex/tooling/tests/unit/push-db-s3.bats b/.vortex/tooling/tests/unit/push-db-s3.bats index 95a5ca50f8..eadb814c22 100644 --- a/.vortex/tooling/tests/unit/push-db-s3.bats +++ b/.vortex/tooling/tests/unit/push-db-s3.bats @@ -658,3 +658,43 @@ setup_s3_push() { popd >/dev/null } + +@test "push-db-s3: Push an indexed database to the primary push bucket" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + mkdir -p .data + echo "migration content" >.data/db2.sql + + unset S3_ACCESS_KEY S3_SECRET_KEY S3_BUCKET S3_REGION S3_PREFIX VORTEX_PUSH_DB_FILE_SUFFIX VORTEX_PUSH_DB_TIMESTAMPED_COPY + export VORTEX_PUSH_DB_S3_ACCESS_KEY="push-access-key" + export VORTEX_PUSH_DB_S3_SECRET_KEY="push-secret-key" + export VORTEX_PUSH_DB_S3_BUCKET="push-bucket" + export VORTEX_PUSH_DB_S3_REGION="eu-west-1" + export VORTEX_PUSH_DB_S3_PREFIX="gha" + export VORTEX_DB_INDEX=2 + export VORTEX_FETCH_DB2_S3_ACCESS_KEY="" + export VORTEX_FETCH_DB2_S3_SECRET_KEY="" + export VORTEX_FETCH_DB2_S3_BUCKET="" + export VORTEX_FETCH_DB2_S3_REGION="" + export VORTEX_FETCH_DB2_S3_PREFIX="" + export VORTEX_FETCH_DB2_FILE="db2.sql" + unset VORTEX_FETCH_DB2_S3_DB_FILE + + mock_curl=$(mock_command "curl") + mock_set_output "${mock_curl}" "200" 1 + + run .vortex/tooling/src/vortex-push-db-s3 + assert_success + # The bucket and the prefix are shared, while the file stays the indexed + # database's own. + assert_output_contains "Local file: ./.data/db2.sql" + assert_output_contains "Remote file: gha/db2.sql" + assert_output_contains "S3 bucket: push-bucket" + assert_output_contains "[ OK ] Finished database 2 dump push to S3." + + assert_string_contains "$(mock_get_call_args "${mock_curl}" 1)" "https://push-bucket.s3.eu-west-1.amazonaws.com/gha/db2.sql" + + rm -rf .data + + popd >/dev/null +} From 0f30d1b049d6f6ce382929d3083b12f2d75fcbf6 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 09:25:56 +1100 Subject: [PATCH 30/40] Used 'filename' in the database storage docs to pass the prose check. --- .vortex/docs/content/development/database-storage.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index 839cbaf496..e8acc68ce0 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -145,8 +145,8 @@ it fetched. The migration database reads its own list in `VORTEX_PUSH_DB2_DESTINATIONS`. Its connection values, such as `VORTEX_PUSH_DB2_FTP_HOST`, fall back to the primary push values, so both -databases can share 1 server or bucket. Each database keeps its own file -name. +databases can share 1 server or bucket. Each database keeps its own +filename. ## Set up an FTP dump store From 06af396e6fc2bc1e47e3251cc3a24423cb0cac20 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 09:32:20 +1100 Subject: [PATCH 31/40] Updated snapshots. --- .../code_provider_other/.circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml index f56bdf626c..a863608ae8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml @@ -289,6 +289,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -307,6 +317,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Refresh the database dumps in the fetch sources that are dump stores. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 From 4587ad8b166cce22935f5f4752b3c76a4180a188 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 10:00:47 +1100 Subject: [PATCH 32/40] Re-triggered CI to clear a flaky check. From 5e5e42f2662623fc3c9132df789a69182ba18518 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 18:08:25 +1100 Subject: [PATCH 33/40] [#3340] Added 'VORTEX_FETCH_DB_STORE' to fetch the database from a sanitized store that the nightly or a manual CI refresh fills from the source. --- .circleci/config.yml | 22 ++++- .env | 44 +++++----- .github/workflows/build-test-deploy.yml | 17 ++-- .github/workflows/vortex-test-didi.yml | 8 +- .vortex/tooling/src/vortex-fetch-db | 46 +++++++---- .vortex/tooling/src/vortex-push-db | 47 +++++------ .vortex/tooling/src/vortex-refresh-db | 19 ++--- .vortex/tooling/tests/_helper.bash | 2 + .vortex/tooling/tests/unit/fetch-db.bats | 95 ++++++++++++++++------ .vortex/tooling/tests/unit/push-db.bats | 56 +++++++------ .vortex/tooling/tests/unit/refresh-db.bats | 34 ++++---- 11 files changed, 230 insertions(+), 160 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ec242ba7f..d261018af2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,15 @@ #; and will be removed during installation or update of Vortex. version: '2.1' +#;< !PROVISION_TYPE_PROFILE +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false +#;> !PROVISION_TYPE_PROFILE + aliases: #;< !PROVISION_TYPE_PROFILE # SSH key fingerprint to fetch the database. @@ -408,7 +417,7 @@ jobs: # Deploy container image. VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED: 1 #;> DB_IN_IMAGE - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 #;< VORTEX_DEV # Tell this provider's demo database dumps apart on the test FTP server @@ -791,6 +800,10 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + #;< !PROVISION_TYPE_PROFILE + when: + not: << pipeline.parameters.refresh_db >> + #;> !PROVISION_TYPE_PROFILE jobs: #;< !PROVISION_TYPE_PROFILE - database: @@ -891,4 +904,11 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly #;> !PROVISION_TYPE_PROFILE diff --git a/.env b/.env index 9356e39918..1bf6f9c4d9 100644 --- a/.env +++ b/.env @@ -263,22 +263,18 @@ VORTEX_FETCH_DB_S3_BUCKET= VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 #;> DB_FETCH_SOURCE_S3 -#;< DB_PUSH -# Upstream source of the database dump file, such as url, acquia or lagoon. -# -# Uncomment to make the database fetch source a dump store: the nightly CI -# job fetches the dump from this source, sanitizes it and pushes it to the -# fetch source as the latest dump. The fetch script of this source reads its -# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep -# secrets in CI. -# VORTEX_REFRESH_DB_SOURCE=url -#;> DB_PUSH - -# Destinations the nightly CI job pushes the sanitized database dump to: -# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source -# when it is a dump store. Each destination reads its own variables, such as -# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 -# source. +# Database store of the sanitized database dump file: ftp or s3. +# +# When set, the database is fetched from the store instead of the source. The +# nightly CI job fetches from the source, sanitizes the database and pushes +# the dump to the store. The store reads the FTP or S3 variables above. Leave +# empty to fetch from the source. +VORTEX_FETCH_DB_STORE= + +# Destinations the nightly CI job pushes the sanitized database dump to +# besides the database store: ftp, s3 or both, such as "ftp,s3". Each +# destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for ftp. +# The database is always fetched from 1 place. # VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 # Also push a copy of each dump named with the time of the push, such as @@ -365,18 +361,16 @@ VORTEX_FETCH_DB2_S3_BUCKET= VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 #;> MIGRATION_DB_FETCH_SOURCE_S3 -#;< MIGRATION_DB_PUSH -# Upstream source of the second database dump file, such as url or lagoon. +# Database store of the second database dump file: ftp or s3. # -# Uncomment to make the second database fetch source a dump store: the -# nightly CI job fetches the dump from this source and pushes it, as fetched, -# to the fetch source, next to the primary dump. -# VORTEX_REFRESH_DB2_SOURCE=url -#;> MIGRATION_DB_PUSH +# When set, the second database is fetched from the store instead of the +# source. The nightly CI job fetches from the source and pushes the dump, as +# fetched, to the store, next to the primary dump. Leave empty to fetch from +# the source. +VORTEX_FETCH_DB2_STORE= # Destinations the nightly CI job pushes the second database dump to, as -# fetched: ftp, s3 or both. Defaults to the second database fetch source when -# it is a dump store. +# fetched, besides the database store: ftp, s3 or both. # VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 # Environment to fetch the second database from. diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index a74d249eae..ffb9793094 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -61,6 +61,13 @@ on: description: 'Enable terminal session for CI jobs' required: false default: false + #;< !PROVISION_TYPE_PROFILE + refresh_db: + type: boolean + description: 'Refresh the database as on the nightly run, without building or deploying' + required: false + default: false + #;> !PROVISION_TYPE_PROFILE #;< !PROVISION_TYPE_PROFILE schedule: - cron: '0 18 * * *' @@ -84,7 +91,7 @@ jobs: lint: runs-on: ubuntu-latest #;< !PROVISION_TYPE_PROFILE - if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} + if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} #;> !PROVISION_TYPE_PROFILE container: @@ -302,7 +309,7 @@ jobs: run: ./scripts/vortex-tooling.sh - name: Adjust variables for a scheduled run - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' || inputs.refresh_db run: | { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" @@ -313,7 +320,7 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" @@ -412,7 +419,7 @@ jobs: runs-on: ubuntu-latest #;< !PROVISION_TYPE_PROFILE needs: database - if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} + if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} #;> !PROVISION_TYPE_PROFILE permissions: @@ -770,7 +777,7 @@ jobs: runs-on: ubuntu-latest needs: [build, lint] #;< !PROVISION_TYPE_PROFILE - if: ${{ !cancelled() && (inputs.deploy_target || (success() && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} + if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} #;> !PROVISION_TYPE_PROFILE permissions: diff --git a/.github/workflows/vortex-test-didi.yml b/.github/workflows/vortex-test-didi.yml index af04199f88..f6c9132f3f 100644 --- a/.github/workflows/vortex-test-didi.yml +++ b/.github/workflows/vortex-test-didi.yml @@ -150,7 +150,7 @@ jobs: run: ./scripts/vortex-tooling.sh - name: Adjust variables for a scheduled run - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' || inputs.refresh_db run: | { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" @@ -161,7 +161,7 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" @@ -724,7 +724,7 @@ jobs: run: ./scripts/vortex-tooling.sh - name: Adjust variables for a scheduled run - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' || inputs.refresh_db run: | { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" @@ -735,7 +735,7 @@ jobs: # Deploy container image. echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" #;> DB_IN_IMAGE - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" diff --git a/.vortex/tooling/src/vortex-fetch-db b/.vortex/tooling/src/vortex-fetch-db index b3f93c2f6e..27025ad873 100755 --- a/.vortex/tooling/src/vortex-fetch-db +++ b/.vortex/tooling/src/vortex-fetch-db @@ -23,10 +23,22 @@ _db_index="${VORTEX_DB_INDEX:-}" _v="VORTEX_FETCH_DB${_db_index}_SOURCE" VORTEX_FETCH_DB_SOURCE="${!_v:-url}" +# Sanitized store of the database dump file: 'ftp' or 's3'. +# +# When set, the database is fetched from the store instead of the source. The +# nightly CI job fetches from the source, sanitizes the database and pushes +# the dump to the store. Leave empty to fetch from the source. +_v="VORTEX_FETCH_DB${_db_index}_STORE" +VORTEX_FETCH_DB_STORE="${!_v:-}" + # Force DB fetch even if the cache exists. _v="VORTEX_FETCH_DB${_db_index}_FORCE" VORTEX_FETCH_DB_FORCE="${!_v:-}" +# Fetch a fresh database backup from the source. +_v="VORTEX_FETCH_DB${_db_index}_FRESH" +VORTEX_FETCH_DB_FRESH="${!_v:-}" + # Proceed with fetch. _v="VORTEX_FETCH_DB${_db_index}_PROCEED" VORTEX_FETCH_DB_PROCEED="${!_v:-1}" @@ -55,22 +67,28 @@ info "Started database${_db_index:+ ${_db_index}} fetch." [ "${VORTEX_FETCH_DB_PROCEED}" != "1" ] && pass "Skipped database fetch as VORTEX_FETCH_DB_PROCEED is not set to 1." && exit 0 -# An 'ftp' or 's3' source with an upstream source in VORTEX_REFRESH_DB_SOURCE -# is a dump store: it receives the dump that the nightly refresh pushes, so -# with a push requested the dump comes from the upstream source. The store -# may hold no dump yet on the first push. -_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" -refresh_source="${!_v:-}" -if [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ] && [ -n "${refresh_source}" ]; then +# A push refreshes the store from the source, so with a push requested the +# database is fetched from the source. The store may hold no dump yet on the +# first push. +if [ -n "${VORTEX_FETCH_DB_STORE}" ]; then + case "${VORTEX_FETCH_DB_STORE}" in + ftp | s3) ;; + *) fail "Unsupported database store \"${VORTEX_FETCH_DB_STORE}\" in VORTEX_FETCH_DB${_db_index}_STORE. Use one of: ftp, s3." ;; + esac + case "${VORTEX_FETCH_DB_SOURCE}" in - ftp | s3) - case "${refresh_source}" in - "${VORTEX_FETCH_DB_SOURCE}" | container_registry | none) fail "The database dump file pushed to ${VORTEX_FETCH_DB_SOURCE} is fetched from an upstream source, but VORTEX_REFRESH_DB${_db_index}_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." ;; - esac - note "Fetching from the ${refresh_source} source in VORTEX_REFRESH_DB${_db_index}_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." - VORTEX_FETCH_DB_SOURCE="${refresh_source}" - ;; + "${VORTEX_FETCH_DB_STORE}" | container_registry | none) fail "The database store in VORTEX_FETCH_DB${_db_index}_STORE is refreshed from a database dump file, but VORTEX_FETCH_DB${_db_index}_SOURCE is \"${VORTEX_FETCH_DB_SOURCE}\". Set the source to another source of the dump file: url, ftp, acquia, lagoon or s3." ;; esac + + if [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then + note "Fetching from the ${VORTEX_FETCH_DB_SOURCE} source to refresh the ${VORTEX_FETCH_DB_STORE} store as VORTEX_PUSH_DB_PROCEED is set to 1." + else + note "Fetching from the ${VORTEX_FETCH_DB_STORE} store in VORTEX_FETCH_DB${_db_index}_STORE." + # The store holds the dump of its last refresh, so a fresh backup needs + # a refresh of the store first. + [ "${VORTEX_FETCH_DB_FRESH}" = "1" ] && note "The ${VORTEX_FETCH_DB_STORE} store holds the database from its last refresh. Run the database refresh in CI to store a fresh database first." + VORTEX_FETCH_DB_SOURCE="${VORTEX_FETCH_DB_STORE}" + fi fi # The fetch script is resolved before the existing dump check, so an diff --git a/.vortex/tooling/src/vortex-push-db b/.vortex/tooling/src/vortex-push-db index 2035c75a87..25760989cf 100755 --- a/.vortex/tooling/src/vortex-push-db +++ b/.vortex/tooling/src/vortex-push-db @@ -4,11 +4,10 @@ # # This is a router script to call relevant scripts based on type. # -# The database dump file is pushed to every destination in -# VORTEX_PUSH_DB_DESTINATIONS, such as 'ftp,s3'. When it is empty and the -# 'ftp' or 's3' fetch source has an upstream source in -# VORTEX_REFRESH_DB_SOURCE (a dump store), the dump is pushed to the fetch -# source. The database is always fetched from 1 source. +# The database dump file is pushed to the database store in +# VORTEX_FETCH_DB_STORE and to every destination in +# VORTEX_PUSH_DB_DESTINATIONS, such as 'ftp,s3'. The database is always +# fetched from 1 place. # # With VORTEX_DB_INDEX set (e.g., "2" for the migration database), the # indexed database pushes its own dump file with the indexed variables, so @@ -33,11 +32,11 @@ _db_index="${VORTEX_DB_INDEX:-}" # and never from a feature branch build. VORTEX_PUSH_DB_PROCEED="${VORTEX_PUSH_DB_PROCEED:-}" -# Destinations to push the database dump file to: 'ftp', 's3' or both, -# separated by commas or spaces. +# Destinations to push the database dump file to besides the database store: +# 'ftp', 's3' or both, separated by commas or spaces. # -# Defaults to the fetch source when it is a dump store. Each destination reads -# its own variables, such as VORTEX_PUSH_DB_FTP_HOST for 'ftp'. +# Each destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST +# for 'ftp'. _v="VORTEX_PUSH_DB${_db_index}_DESTINATIONS" VORTEX_PUSH_DB_DESTINATIONS="${!_v:-}" @@ -45,9 +44,9 @@ VORTEX_PUSH_DB_DESTINATIONS="${!_v:-}" _v="VORTEX_FETCH_DB${_db_index}_SOURCE" VORTEX_FETCH_DB_SOURCE="${!_v:-url}" -# Upstream source of the pushed database dump file. -_v="VORTEX_REFRESH_DB${_db_index}_SOURCE" -VORTEX_REFRESH_DB_SOURCE="${!_v:-}" +# Database store, which receives the pushed database dump file. +_v="VORTEX_FETCH_DB${_db_index}_STORE" +VORTEX_FETCH_DB_STORE="${!_v:-}" # ------------------------------------------------------------------------------ @@ -63,30 +62,24 @@ info "Started database${_db_index:+ ${_db_index}} push." [ "${VORTEX_PUSH_DB_PROCEED}" != "1" ] && pass "Skipped database push as VORTEX_PUSH_DB_PROCEED is not set to 1." && exit 0 -# A dump store receives the dump that the nightly refresh exports. -if [ -z "${VORTEX_PUSH_DB_DESTINATIONS}" ] && [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then - case "${VORTEX_FETCH_DB_SOURCE}" in - ftp | s3) VORTEX_PUSH_DB_DESTINATIONS="${VORTEX_FETCH_DB_SOURCE}" ;; - esac -fi - destinations=() -for destination in ${VORTEX_PUSH_DB_DESTINATIONS//,/ }; do +for destination in ${VORTEX_FETCH_DB_STORE} ${VORTEX_PUSH_DB_DESTINATIONS//,/ }; do case "${destination}" in ftp | s3) ;; - *) fail "Unsupported database push destination \"${destination}\" in VORTEX_PUSH_DB${_db_index}_DESTINATIONS. Use one or more of: ftp, s3." ;; + *) fail "Unsupported database push destination \"${destination}\" in VORTEX_FETCH_DB${_db_index}_STORE or VORTEX_PUSH_DB${_db_index}_DESTINATIONS. Use one or more of: ftp, s3." ;; esac - # Pushing to the fetch source replaces the dump it fetched with its own - # export, so the source must be a dump store refreshed from upstream. - if [ "${destination}" = "${VORTEX_FETCH_DB_SOURCE}" ] && [ -z "${VORTEX_REFRESH_DB_SOURCE}" ]; then - fail "Refused to push the database to the \"${destination}\" fetch source as VORTEX_REFRESH_DB${_db_index}_SOURCE is not set, so the push would replace the dump it fetched." - fi + # Pushing to the fetch source replaces the dump the database was fetched + # from with its own export. + [ "${destination}" = "${VORTEX_FETCH_DB_SOURCE}" ] && fail "Refused to push the database to the \"${destination}\" fetch source, as the push would replace the dump the database was fetched from." + + # The store can also be listed as a destination. + [[ " ${destinations[*]:-} " == *" ${destination} "* ]] && continue destinations+=("${destination}") done -[ "${#destinations[@]}" -eq 0 ] && pass "Skipped database push as VORTEX_PUSH_DB${_db_index}_DESTINATIONS is empty and the \"${VORTEX_FETCH_DB_SOURCE}\" fetch source is not a dump store." && exit 0 +[ "${#destinations[@]}" -eq 0 ] && pass "Skipped database push as VORTEX_FETCH_DB${_db_index}_STORE and VORTEX_PUSH_DB${_db_index}_DESTINATIONS are empty." && exit 0 # The pushed dump leaves the hosting and CI, so it must be sanitized. Only the # primary database is sanitized: an indexed database is pushed as fetched. diff --git a/.vortex/tooling/src/vortex-refresh-db b/.vortex/tooling/src/vortex-refresh-db index b679c3bdc0..4fd60c2f2b 100755 --- a/.vortex/tooling/src/vortex-refresh-db +++ b/.vortex/tooling/src/vortex-refresh-db @@ -15,10 +15,10 @@ # the service is exported and pushed as VORTEX_DB2_IMAGE, without # provisioning. This runs only when VORTEX_REFRESH_DB2_PROCEED is set to 1. # -# With VORTEX_PUSH_DB_PROCEED set to 1 and a push destination, either in -# VORTEX_PUSH_DB_DESTINATIONS or an 'ftp' or 's3' fetch source with an -# upstream source in VORTEX_REFRESH_DB_SOURCE (a dump store), the database is -# sanitized after provisioning, so 'vortex-push-db' pushes a sanitized dump. +# With VORTEX_PUSH_DB_PROCEED set to 1 and a push destination, either the +# database store in VORTEX_FETCH_DB_STORE or a destination in +# VORTEX_PUSH_DB_DESTINATIONS, the database is sanitized after provisioning, +# so 'vortex-push-db' pushes a sanitized dump. # # Every variable of the calling environment is passed to the provisioning in # the 'cli' container, so the script is intended to run in CI. @@ -50,11 +50,6 @@ VORTEX_DB_IMAGE_BASE="${VORTEX_DB_IMAGE_BASE:-}" # set: the database image is built from a dump file, so it needs a dump file # source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed # database uses VORTEX_REFRESH_DB2_SOURCE. -# -# With an 'ftp' or 's3' fetch source, setting this value makes the fetch -# source a dump store: with VORTEX_PUSH_DB_PROCEED set to 1, 'vortex-fetch-db' -# fetches the dump from this source and 'vortex-push-db' pushes the exported -# dump to the store. _v="VORTEX_REFRESH_DB${_db_index}_SOURCE" VORTEX_REFRESH_DB_SOURCE="${!_v:-}" @@ -112,12 +107,8 @@ rebuild_image=0 # requested the exported dump must be sanitized. refresh_store=0 if [ -z "${_db_index}" ] && [ -z "${db_image}" ] && [ "${VORTEX_PUSH_DB_PROCEED:-}" = "1" ]; then - if [ -n "${VORTEX_PUSH_DB_DESTINATIONS:-}" ]; then + if [ -n "${VORTEX_FETCH_DB_STORE:-}" ] || [ -n "${VORTEX_PUSH_DB_DESTINATIONS:-}" ]; then refresh_store=1 - elif [ -n "${VORTEX_REFRESH_DB_SOURCE}" ]; then - case "${fetch_source}" in - ftp | s3) refresh_store=1 ;; - esac fi fi diff --git a/.vortex/tooling/tests/_helper.bash b/.vortex/tooling/tests/_helper.bash index 9da6a39932..647df8cb75 100644 --- a/.vortex/tooling/tests/_helper.bash +++ b/.vortex/tooling/tests/_helper.bash @@ -120,6 +120,8 @@ setup() { unset VORTEX_FETCH_DB_SOURCE unset VORTEX_DB_IMAGE unset VORTEX_FETCH_DB_FORCE + unset VORTEX_FETCH_DB_FRESH + unset VORTEX_FETCH_DB_STORE unset VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED unset VORTEX_PUSH_DB_PROCEED unset VORTEX_PUSH_DB_DESTINATIONS diff --git a/.vortex/tooling/tests/unit/fetch-db.bats b/.vortex/tooling/tests/unit/fetch-db.bats index 0ab820b840..6ee7dfed98 100644 --- a/.vortex/tooling/tests/unit/fetch-db.bats +++ b/.vortex/tooling/tests/unit/fetch-db.bats @@ -325,71 +325,96 @@ EOF popd >/dev/null } -@test "fetch-db: Fetch a dump store source from the refresh source with a push requested" { +@test "fetch-db: Fetch from the store without a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_PROCEED="1" + unset VORTEX_PUSH_DB_PROCEED + + for db_store in ftp s3; do + export VORTEX_FETCH_DB_STORE="${db_store}" + + run .vortex/tooling/src/vortex-fetch-db + assert_success + assert_output_contains "Fetching from the ${db_store} store in VORTEX_FETCH_DB_STORE." + assert_output_contains "Fetched from ${db_store}." + assert_output_not_contains "Fetched from lagoon." + assert_output_not_contains "last refresh" + assert_output_contains "Finished database fetch." + done + + popd >/dev/null +} + +@test "fetch-db: Fetch from the source to refresh the store with a push requested" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_SOURCE="lagoon" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" - export VORTEX_REFRESH_DB_SOURCE="lagoon" - for db_source in ftp s3; do - export VORTEX_FETCH_DB_SOURCE="${db_source}" + for db_store in ftp s3; do + export VORTEX_FETCH_DB_STORE="${db_store}" run .vortex/tooling/src/vortex-fetch-db assert_success - assert_output_contains "Fetching from the lagoon source in VORTEX_REFRESH_DB_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." + assert_output_contains "Fetching from the lagoon source to refresh the ${db_store} store as VORTEX_PUSH_DB_PROCEED is set to 1." assert_output_contains "Fetched from lagoon." - assert_output_not_contains "Fetched from ${db_source}." + assert_output_not_contains "Fetched from ${db_store}." assert_output_contains "Finished database fetch." done popd >/dev/null } -@test "fetch-db: Fetch an indexed dump store source from its refresh source with a push requested" { +@test "fetch-db: Fetch an indexed database from its own store" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts export VORTEX_DB_INDEX=3 - export VORTEX_FETCH_DB3_SOURCE="ftp" + export VORTEX_FETCH_DB3_SOURCE="url" + export VORTEX_FETCH_DB3_STORE="s3" export VORTEX_FETCH_DB3_PROCEED="1" export VORTEX_FETCH_DB3_DIR="${BUILD_DIR}/db3" - export VORTEX_PUSH_DB_PROCEED="1" - export VORTEX_REFRESH_DB3_SOURCE="url" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" + unset VORTEX_PUSH_DB_PROCEED run .vortex/tooling/src/vortex-fetch-db assert_success - assert_output_contains "Fetching from the url source in VORTEX_REFRESH_DB3_SOURCE as VORTEX_PUSH_DB_PROCEED is set to 1." - assert_output_contains "Fetched from url." + assert_output_contains "Fetching from the s3 store in VORTEX_FETCH_DB3_STORE." + assert_output_contains "Fetched from s3." assert_output_not_contains "Fetched from ftp." - assert_output_not_contains "Fetched from lagoon." + assert_output_not_contains "Fetched from url." popd >/dev/null } -@test "fetch-db: Fetch a dump store source from the store without a push requested" { +@test "fetch-db: Point a fresh fetch from the store to the refresh in CI" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" export VORTEX_FETCH_DB_PROCEED="1" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_FRESH="1" unset VORTEX_PUSH_DB_PROCEED run .vortex/tooling/src/vortex-fetch-db assert_success + assert_output_contains "The ftp store holds the database from its last refresh. Run the database refresh in CI to store a fresh database first." assert_output_contains "Fetched from ftp." - assert_output_not_contains "Fetched from lagoon." popd >/dev/null } -@test "fetch-db: Fetch an ftp source without a refresh source as an upstream source" { +@test "fetch-db: Fetch from the source without a store" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts @@ -397,7 +422,7 @@ EOF export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_FETCH_DB_PROCEED="1" export VORTEX_PUSH_DB_PROCEED="1" - export VORTEX_REFRESH_DB_SOURCE="" + export VORTEX_FETCH_DB_STORE="" run .vortex/tooling/src/vortex-fetch-db assert_success @@ -407,23 +432,39 @@ EOF popd >/dev/null } -@test "fetch-db: Fail with an invalid upstream source for a dump store source with a push requested" { +@test "fetch-db: Fail with an unsupported store" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_fetch_scripts - export VORTEX_FETCH_DB_SOURCE="ftp" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="gdrive" + export VORTEX_FETCH_DB_PROCEED="1" + + run .vortex/tooling/src/vortex-fetch-db + assert_failure + assert_output_contains '[FAIL] Unsupported database store "gdrive" in VORTEX_FETCH_DB_STORE. Use one of: ftp, s3.' + assert_output_not_contains "Fetched from" + + popd >/dev/null +} + +@test "fetch-db: Fail when the store cannot be refreshed from the source" { + pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 + + stub_fetch_scripts + + export VORTEX_FETCH_DB_STORE="ftp" export VORTEX_FETCH_DB_PROCEED="1" - export VORTEX_PUSH_DB_PROCEED="1" export VORTEX_FETCH_DB_SEMAPHORE="${BUILD_DIR}/fetch-db-fresh" - for refresh_source in ftp container_registry none; do - export VORTEX_REFRESH_DB_SOURCE="${refresh_source}" + for db_source in ftp container_registry none; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-fetch-db assert_failure - assert_output_contains "[FAIL] The database dump file pushed to ftp is fetched from an upstream source, but VORTEX_REFRESH_DB_SOURCE is \"${refresh_source}\". Set it to another source of the dump file: url, ftp, acquia, lagoon or s3." - assert_output_not_contains "Fetched from ftp." + assert_output_contains "[FAIL] The database store in VORTEX_FETCH_DB_STORE is refreshed from a database dump file, but VORTEX_FETCH_DB_SOURCE is \"${db_source}\". Set the source to another source of the dump file: url, ftp, acquia, lagoon or s3." + assert_output_not_contains "Fetched from" assert_output_not_contains "Finished database fetch." assert_file_not_exists "${BUILD_DIR}/fetch-db-fresh" done diff --git a/.vortex/tooling/tests/unit/push-db.bats b/.vortex/tooling/tests/unit/push-db.bats index c83bbe8446..8a6f06ab0a 100644 --- a/.vortex/tooling/tests/unit/push-db.bats +++ b/.vortex/tooling/tests/unit/push-db.bats @@ -37,26 +37,26 @@ stub_push_scripts() { popd >/dev/null } -@test "push-db: Push to the dump store fetch source by default" { +@test "push-db: Push to the store by default" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" unset VORTEX_PUSH_DB_DESTINATIONS - for db_source in ftp s3; do - export VORTEX_FETCH_DB_SOURCE="${db_source}" + for db_store in ftp s3; do + export VORTEX_FETCH_DB_STORE="${db_store}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "Pushed to ${db_source} with index []." + assert_output_contains "Pushed to ${db_store} with index []." assert_output_contains "[ OK ] Finished database push." - for other_source in ftp s3; do - [ "${other_source}" = "${db_source}" ] && continue - assert_output_not_contains "Pushed to ${other_source}" + for other_store in ftp s3; do + [ "${other_store}" = "${db_store}" ] && continue + assert_output_not_contains "Pushed to ${other_store}" done done @@ -70,7 +70,6 @@ stub_push_scripts() { export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_FETCH_DB_SOURCE="lagoon" - unset VORTEX_REFRESH_DB_SOURCE for destinations in "ftp,s3" "ftp s3" " ftp , s3 "; do export VORTEX_PUSH_DB_DESTINATIONS="${destinations}" @@ -87,20 +86,21 @@ stub_push_scripts() { popd >/dev/null } -@test "push-db: Push to the dump store fetch source and another destination" { +@test "push-db: Push to the store and another destination" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" export VORTEX_PUSH_DB_DESTINATIONS="ftp,s3" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "Pushed to ftp with index []." assert_output_contains "Pushed to s3 with index []." + # The store listed as a destination too is pushed to once. + assert_equal "1" "$(grep -c "Pushed to ftp with index \[\]." <<<"${output}")" popd >/dev/null } @@ -113,7 +113,8 @@ stub_push_scripts() { export VORTEX_DB_INDEX=2 export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_FETCH_DB2_SOURCE="url" - export VORTEX_PUSH_DB2_DESTINATIONS="s3" + export VORTEX_FETCH_DB2_STORE="s3" + export VORTEX_FETCH_DB_STORE="ftp" export VORTEX_PUSH_DB_DESTINATIONS="ftp" # The indexed database is pushed as fetched, so the sanitization setting # does not apply to it. @@ -126,6 +127,13 @@ stub_push_scripts() { assert_output_not_contains "Pushed to ftp" assert_output_contains "[ OK ] Finished database 2 push." + export VORTEX_PUSH_DB2_DESTINATIONS="ftp" + + run .vortex/tooling/src/vortex-push-db + assert_success + assert_output_contains "Pushed to s3 with index [2]." + assert_output_contains "Pushed to ftp with index [2]." + popd >/dev/null } @@ -135,17 +143,14 @@ stub_push_scripts() { stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 - unset VORTEX_PUSH_DB_DESTINATIONS + unset VORTEX_PUSH_DB_DESTINATIONS VORTEX_FETCH_DB_STORE - # A fetch source that is not a dump store, or a dump store source with no - # upstream source, gives no destination. - for case in "url:lagoon" "lagoon:lagoon" "container_registry:lagoon" "ftp:" "s3:"; do - export VORTEX_FETCH_DB_SOURCE="${case%%:*}" - export VORTEX_REFRESH_DB_SOURCE="${case#*:}" + for db_source in url lagoon container_registry ftp s3; do + export VORTEX_FETCH_DB_SOURCE="${db_source}" run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains "[ OK ] Skipped database push as VORTEX_PUSH_DB_DESTINATIONS is empty and the \"${case%%:*}\" fetch source is not a dump store." + assert_output_contains "[ OK ] Skipped database push as VORTEX_FETCH_DB_STORE and VORTEX_PUSH_DB_DESTINATIONS are empty." assert_output_not_contains "Pushed to" assert_output_not_contains "Finished database push." done @@ -163,7 +168,7 @@ stub_push_scripts() { run .vortex/tooling/src/vortex-push-db assert_success - assert_output_contains '[ OK ] Skipped database push as VORTEX_PUSH_DB_DESTINATIONS is empty and the "url" fetch source is not a dump store.' + assert_output_contains '[ OK ] Skipped database push as VORTEX_FETCH_DB_STORE and VORTEX_PUSH_DB_DESTINATIONS are empty.' assert_output_not_contains "Pushed to" popd >/dev/null @@ -180,26 +185,25 @@ stub_push_scripts() { run .vortex/tooling/src/vortex-push-db assert_failure - assert_output_contains '[FAIL] Unsupported database push destination "gdrive" in VORTEX_PUSH_DB_DESTINATIONS. Use one or more of: ftp, s3.' + assert_output_contains '[FAIL] Unsupported database push destination "gdrive" in VORTEX_FETCH_DB_STORE or VORTEX_PUSH_DB_DESTINATIONS. Use one or more of: ftp, s3.' # Every destination is checked before the first push. assert_output_not_contains "Pushed to" popd >/dev/null } -@test "push-db: Refuse to push to the fetch source without an upstream source" { +@test "push-db: Refuse to push to the fetch source" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 stub_push_scripts export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="" export VORTEX_PUSH_DB_DESTINATIONS="s3,ftp" run .vortex/tooling/src/vortex-push-db assert_failure - assert_output_contains '[FAIL] Refused to push the database to the "ftp" fetch source as VORTEX_REFRESH_DB_SOURCE is not set, so the push would replace the dump it fetched.' + assert_output_contains '[FAIL] Refused to push the database to the "ftp" fetch source, as the push would replace the dump the database was fetched from.' assert_output_not_contains "Pushed to" popd >/dev/null diff --git a/.vortex/tooling/tests/unit/refresh-db.bats b/.vortex/tooling/tests/unit/refresh-db.bats index 87ad5353cc..e0e6996abd 100644 --- a/.vortex/tooling/tests/unit/refresh-db.bats +++ b/.vortex/tooling/tests/unit/refresh-db.bats @@ -246,7 +246,7 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Sanitizes the database for a dump store source with a push requested" { +@test "refresh-db: Sanitizes the database for a store with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data @@ -259,8 +259,8 @@ load ../_helper.bash stub_sibling "vortex-fetch-db" "Fetched the dump file." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" mock_command "sleep" >/dev/null mock_docker=$(mock_command "docker") @@ -273,7 +273,7 @@ load ../_helper.bash run .vortex/tooling/src/vortex-refresh-db assert_success - # The fetch step already fetched the dump from the upstream source. + # The fetch step already fetched the dump from the source. assert_output_not_contains "Fetched the dump file." assert_output_contains "[ OK ] Copied database dump file ./.data/db.sql into the cli container." assert_output_contains "Sanitized the database." @@ -290,15 +290,15 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Exports the dump store database unsanitized when a push is not requested" { +@test "refresh-db: Exports the store database unsanitized when a push is not requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "store" >.data/db.sql unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_PUSH_DB_PROCEED - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" stub_sibling "vortex-login-container-registry" "Logged in to the registry." stub_sibling "vortex-export-db" 'Exported the database to ${1}.' @@ -324,7 +324,7 @@ load ../_helper.bash mkdir -p .data echo "upstream" >.data/db.sql - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_FETCH_DB_STORE export VORTEX_FETCH_DB_SOURCE="lagoon" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PUSH_DB_DESTINATIONS="ftp,s3" @@ -345,13 +345,13 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Exports an ftp source database unsanitized without a refresh source" { +@test "refresh-db: Exports the database unsanitized without a store or destinations" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "upstream" >.data/db.sql - unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_REFRESH_DB_SOURCE + unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE VORTEX_FETCH_DB_STORE VORTEX_PUSH_DB_DESTINATIONS export VORTEX_FETCH_DB_SOURCE="ftp" export VORTEX_PUSH_DB_PROCEED=1 @@ -365,18 +365,18 @@ load ../_helper.bash assert_success assert_output_contains "[ OK ] Finished database refresh." - # Without a refresh source, the ftp source is not a dump store. + # Nothing is pushed, so the database is not sanitized. assert_equal "5" "$(mock_get_call_num "${mock_docker}")" popd >/dev/null } -@test "refresh-db: Fails when the sanitization is skipped for a dump store source with a push requested" { +@test "refresh-db: Fails when the sanitization is skipped for a store with a push requested" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="ftp" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="ftp" export VORTEX_PUSH_DB_PROCEED=1 export VORTEX_PROVISION_SANITIZE_DB_SKIP=1 @@ -393,15 +393,15 @@ load ../_helper.bash popd >/dev/null } -@test "refresh-db: Fails when the sanitization fails for a dump store source" { +@test "refresh-db: Fails when the sanitization fails for a store" { pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1 mkdir -p .data echo "upstream" >.data/db.sql unset VORTEX_DB_IMAGE VORTEX_DB_IMAGE_BASE - export VORTEX_FETCH_DB_SOURCE="s3" - export VORTEX_REFRESH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_SOURCE="lagoon" + export VORTEX_FETCH_DB_STORE="s3" export VORTEX_PUSH_DB_PROCEED=1 stub_sibling "vortex-login-container-registry" "Logged in to the registry." From 3daeddbf9950dfbff8195270bdcbec6effe1500c Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 18:15:24 +1100 Subject: [PATCH 34/40] [#3340] Documented fetching from a sanitized store, refreshing it on demand in CI and fetching from the source directly. --- .github/workflows/build-test-deploy.yml | 2 +- .../content/continuous-integration/README.mdx | 8 +- .../contributing/maintenance/installer.mdx | 1 + .../content/development/database-storage.mdx | 158 +++++++++++------- .../docs/content/development/migrations.mdx | 5 +- .../docs/content/development/provision.mdx | 6 +- .../docs/content/development/variables.mdx | 8 +- .vortex/docs/content/features.mdx | 2 + 8 files changed, 117 insertions(+), 73 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index ffb9793094..ae9cc5ef31 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -64,7 +64,7 @@ on: #;< !PROVISION_TYPE_PROFILE refresh_db: type: boolean - description: 'Refresh the database as on the nightly run, without building or deploying' + description: 'Refresh the database only' required: false default: false #;> !PROVISION_TYPE_PROFILE diff --git a/.vortex/docs/content/continuous-integration/README.mdx b/.vortex/docs/content/continuous-integration/README.mdx index f98c5e15f9..66467478cf 100644 --- a/.vortex/docs/content/continuous-integration/README.mdx +++ b/.vortex/docs/content/continuous-integration/README.mdx @@ -79,9 +79,11 @@ site from: - Caches the dump, so the `build` job and later runs skip the download - On the nightly run of a project that stores its database in a container image, pushes the refreshed image to the registry -- On the nightly run of a project with push destinations or a dump store, - pushes the sanitized dump to the FTP server or S3 bucket (see [Database - storage](../development/database-storage.mdx#fetch-from-a-dump-store)) +- On the nightly run of a project with a database store or push + destinations, pushes the sanitized dump to the FTP server or S3 bucket. + A manual run with the `refresh_db` option does the same on demand (see + [Database + storage](../development/database-storage.mdx#fetch-from-a-store)) The job exists only when the site provisions from a database dump. With the `profile` provision type, the pipeline has no `database` job and no nightly diff --git a/.vortex/docs/content/contributing/maintenance/installer.mdx b/.vortex/docs/content/contributing/maintenance/installer.mdx index aed67ba717..eb7dd0c4e4 100644 --- a/.vortex/docs/content/contributing/maintenance/installer.mdx +++ b/.vortex/docs/content/contributing/maintenance/installer.mdx @@ -225,6 +225,7 @@ these areas, among many others: - **Services**: Solr, Redis, ClamAV (enabled/disabled combinations) - **Deployment**: artifact, webhook, Lagoon, all combined - **Database sources**: Acquia, Lagoon, FTP, S3, URL, container registry +- **Database stores**: FTP, S3, for the primary and the migration database ### Run tests diff --git a/.vortex/docs/content/development/database-storage.mdx b/.vortex/docs/content/development/database-storage.mdx index e8acc68ce0..2c3b6ee486 100644 --- a/.vortex/docs/content/development/database-storage.mdx +++ b/.vortex/docs/content/development/database-storage.mdx @@ -6,24 +6,25 @@ sidebar_position: 6 # Database storage Your site's database reaches a developer's machine and CI in 1 of 3 ways. -The fetch source in `VORTEX_FETCH_DB_SOURCE` picks the way. This page -shows how fetching, refreshing and pushing the database work in each case. -It also covers setting up an FTP server or an S3 bucket as a dump store, -so developers get a sanitized dump without access to your hosting. +`VORTEX_FETCH_DB_SOURCE` sets where the database comes from, such as your +production environment. `VORTEX_FETCH_DB_STORE` adds an optional sanitized +store on an FTP server or in an S3 bucket, so developers get a sanitized +database without access to production. This page shows how fetching, +refreshing and pushing the database work in each case. The page covers these commands: - `ahoy fetch-db` (`vortex-fetch-db`) downloads a dump into `.data` - `vortex-refresh-db` provisions a site from the fetched dump in CI and exports it again -- `vortex-push-db` uploads the exported dump to a dump store on the - nightly CI run +- `vortex-push-db` uploads the exported dump to the store on the nightly + CI run ## Fetch from the source -With the `url`, `ftp`, `acquia`, `lagoon` or `s3` source, every -environment downloads the dump from that source. Developers need -credentials for it. +Without a store, every environment downloads the dump from the `url`, +`ftp`, `acquia`, `lagoon` or `s3` source. Developers need credentials for +it, and `ahoy fetch-db --fresh` asks a hosting source for a new backup. ```text Developer CI (every run with no cached dump) @@ -61,18 +62,17 @@ image](../continuous-integration/README.mdx#dump-file-source-for-a-database-imag Deploy DB image (vortex-push-db-image) ──▶ container registry ``` -## Fetch from a dump store +## Fetch from a store -A dump store is an `ftp` or `s3` source with an upstream source set in -`VORTEX_REFRESH_DB_SOURCE`, such as `lagoon`. Developers and branch builds -fetch from the store and need no hosting access. The nightly CI run -refreshes the store from the upstream source: +With `VORTEX_FETCH_DB_STORE` set to `ftp` or `s3`, the nightly CI run is +the only place that fetches from the source. It sanitizes the database +and pushes the dump to the store: ```text Nightly CI (VORTEX_PUSH_DB_PROCEED=1) ───────────────────────────────────── Fetch DB - │ fetches from VORTEX_REFRESH_DB_SOURCE in place of the store + │ fetches from VORTEX_FETCH_DB_SOURCE, for example lagoon ▼ Export DB (vortex-refresh-db) │ provisions the dump, sanitizes it and exports it @@ -81,31 +81,70 @@ refreshes the store from the upstream source: │ 1. replaces db.sql with the new dump │ 2. with VORTEX_PUSH_DB_TIMESTAMPED_COPY=1, adds a timestamped copy ▼ - FTP server or S3 bucket - ├── db.sql ◀── ahoy fetch-db, branch builds + FTP server or S3 bucket (VORTEX_FETCH_DB_STORE) + ├── db.sql ◀── ahoy fetch-db, CI without a cached dump ├── db_20261110-180000.sql timestamped copies, opt-in └── db_20261111-180000.sql ``` +Every other environment reads from the store: + +| Who | Fetches from | +|---|---| +| Developer running `ahoy fetch-db` | The store | +| CI build during the day | The CI cache, or the store when the cache is empty | +| Nightly or manual refresh in CI | The source, then pushes to the store | + +Only the refresh needs the credentials of the source, so production +access stays in CI. + `db.sql` always holds the latest dump, and fetching downloads it. With a file suffix, the latest dump is the suffixed file, such as `db_gha.sql` (see [Tell pushes apart](#tell-pushes-apart)). By default, it's the only -file a push creates. To keep a history of the -pushed dumps, set `VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push -then also creates a copy named with its UTC time, identical to `db.sql` -at that moment. Nothing removes the copies, so clear out the old ones -yourself. - -Every other CI run fetches from the store, so only the nightly run needs -the credentials of the upstream source. +file a push creates. To keep a history of the pushed dumps, set +`VORTEX_PUSH_DB_TIMESTAMPED_COPY=1` in `.env`. Each push then also creates +a copy named with its UTC time, identical to `db.sql` at that moment. +Nothing removes the copies, so clear out the old ones yourself. The pushed dump leaves your hosting and CI, so it's always sanitized. The -nightly run stops with an error when `VORTEX_PROVISION_SANITIZE_DB_SKIP` -is `1`. +refresh stops with an error when `VORTEX_PROVISION_SANITIZE_DB_SKIP` is +`1`. + +### Refresh the store on demand + +The store holds the database from its last refresh, which is usually the +nightly run. When you need today's database, run the refresh in CI. It +fetches from the source, sanitizes the database and pushes it to the +store, without building or deploying. + +- On GitHub Actions, open the **Database, Build, Test and Deploy** + workflow, click **Run workflow**, select **Refresh the database only** + and click **Run workflow**. +- On CircleCI, click **Trigger Pipeline** on the project, add the + `refresh_db` boolean parameter set to `true` and click + **Trigger Pipeline**. + +After the run finishes, download the new dump: + +```shell +ahoy fetch-db --fresh +``` + +### Fetch from the source directly + +If you have access to the source, you can skip the store. Clear +`VORTEX_FETCH_DB_STORE` for 1 run: + +```shell +VORTEX_FETCH_DB_STORE= ahoy fetch-db --fresh +``` + +The fetch then reads the source variables and credentials, the same as +without a store. The database isn't sanitized, so keep it on your machine. ### Multiple databases -The migration database pushes its own dump file to its own fetch source. +The migration database reads its own store in `VORTEX_FETCH_DB2_STORE`. Its files sit next to the primary dump in the same store: ```text @@ -116,31 +155,25 @@ Its files sit next to the primary dump in the same store: └── db2_20261111-180000.sql ``` -To make the migration source a dump store, set -`VORTEX_REFRESH_DB2_SOURCE` to its upstream source. The -**Push migration DB** step pushes the migration dump as fetched, because -**Vortex** sanitizes only the Drupal database. +The **Push migration DB** step pushes the migration dump as fetched, +because **Vortex** sanitizes only the Drupal database. ### Push to several destinations -The database is always fetched from 1 source, but the nightly run can push -it to several destinations. List them in `VORTEX_PUSH_DB_DESTINATIONS`, -separated by commas or spaces: +The database is always fetched from 1 place, but the refresh can push it +to more destinations than the store. List them in +`VORTEX_PUSH_DB_DESTINATIONS`, separated by commas or spaces: ```shell VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 ``` Each destination reads its own push variables, such as -`VORTEX_PUSH_DB_FTP_HOST` or `VORTEX_PUSH_DB_S3_BUCKET`. When a destination -is also the fetch source, its push variables fall back to the fetch values. -A destination can be pushed to from any fetch source, so a project that -fetches from `lagoon` can keep sanitized copies on FTP and S3. - -When `VORTEX_PUSH_DB_DESTINATIONS` is empty, the push goes to the dump store -fetch source. The push refuses to push to the fetch source itself -without `VORTEX_REFRESH_DB_SOURCE`, because the push would replace the dump -it fetched. +`VORTEX_PUSH_DB_FTP_HOST` or `VORTEX_PUSH_DB_S3_BUCKET`, which fall back to +the fetch values. A destination can be pushed to from any source, so a +project without a store can still keep sanitized copies on FTP and S3. +The push refuses a destination that's also the source, because the push +would replace the dump the database was fetched from. The migration database reads its own list in `VORTEX_PUSH_DB2_DESTINATIONS`. Its connection values, such as @@ -148,7 +181,7 @@ The migration database reads its own list in databases can share 1 server or bucket. Each database keeps its own filename. -## Set up an FTP dump store +## Set up an FTP store Before you begin, create an FTP account that can write to a directory on the server. The push uploads the dump under a temporary name, then renames @@ -156,10 +189,12 @@ it into place, so a failed upload keeps the latest dump. The server must allow the `DELE`, `RNFR` and `RNTO` commands, which standard FTP servers do. -1. In `.env`, set the fetch source and the server: +1. In `.env`, set the store and the server. Keep the source of your + database in `VORTEX_FETCH_DB_SOURCE`: ```shell - VORTEX_FETCH_DB_SOURCE=ftp + VORTEX_FETCH_DB_SOURCE=lagoon + VORTEX_FETCH_DB_STORE=ftp VORTEX_FETCH_DB_FTP_HOST=ftp.example.com VORTEX_FETCH_DB_FTP_PORT=21 VORTEX_FETCH_DB_FTP_DIR=your_site @@ -170,18 +205,17 @@ do. the FTP account. Leading and trailing slashes are optional. Leave it empty for the root directory. -2. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your - upstream source, for example `lagoon`. Add the variables and secrets of - that source from the table in [Dump file source for a database - image](../continuous-integration/README.mdx#dump-file-source-for-a-database-image). -3. Add `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` as CI +2. Add `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` as CI secrets. GitHub Actions passes them to the `database` job. CircleCI reads them from the project environment variables. -4. Ask each developer to set `VORTEX_FETCH_DB_FTP_USER` and +3. Ask each developer to set `VORTEX_FETCH_DB_FTP_USER` and `VORTEX_FETCH_DB_FTP_PASS` in `.env.local`. -5. After the next nightly run, confirm that the **Push DB** step ends with +4. After the next nightly run, confirm that the **Push DB** step ends with `Finished database push.` and that the server holds `db.sql`. +The installer sets up steps 1 and 2 when you select **FTP server** at the +**Database store** prompt. + To give CI an account with write access and developers a read-only one, set the CI account in `VORTEX_PUSH_DB_FTP_USER` and `VORTEX_PUSH_DB_FTP_PASS`. On GitHub Actions, map those 2 secrets in the @@ -190,7 +224,7 @@ set the CI account in `VORTEX_PUSH_DB_FTP_USER` and FTP sends the credentials and the dump unencrypted. Use a server on a network you trust, or the S3 store. -## Set up an S3 dump store +## Set up an S3 store Before you begin, create a private S3 bucket with public access blocked. @@ -210,23 +244,23 @@ Before you begin, create a private S3 bucket with public access blocked. } ``` -2. In `.env`, set the fetch source and the bucket: +2. In `.env`, set the store and the bucket. Keep the source of your + database in `VORTEX_FETCH_DB_SOURCE`: ```shell - VORTEX_FETCH_DB_SOURCE=s3 + VORTEX_FETCH_DB_SOURCE=lagoon + VORTEX_FETCH_DB_STORE=s3 VORTEX_FETCH_DB_S3_BUCKET=your-bucket VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 VORTEX_FETCH_DB_S3_PREFIX=your_site ``` -3. In `.env`, uncomment `VORTEX_REFRESH_DB_SOURCE` and set it to your - upstream source, and add the variables and secrets of that source. -4. Add the access key of the IAM user as the +3. Add the access key of the IAM user as the `VORTEX_FETCH_DB_S3_ACCESS_KEY` and `VORTEX_FETCH_DB_S3_SECRET_KEY` CI secrets. -5. Create a second IAM user for developers with `s3:GetObject` only. Ask +4. Create a second IAM user for developers with `s3:GetObject` only. Ask each developer to set its keys in `.env.local`. -6. After the next nightly run, confirm that the **Push DB** step ends with +5. After the next nightly run, confirm that the **Push DB** step ends with `Finished database push.` and that the bucket holds `db.sql`. S3 makes the timestamped copy on its side, so the dump is uploaded once. diff --git a/.vortex/docs/content/development/migrations.mdx b/.vortex/docs/content/development/migrations.mdx index 4d70be8733..139eaa136e 100644 --- a/.vortex/docs/content/development/migrations.mdx +++ b/.vortex/docs/content/development/migrations.mdx @@ -32,7 +32,10 @@ When you run the **Vortex** installer, answer **Yes** at the **Use a second database for migrations?** prompt. Then choose where the migration dump comes from at the **Migration database source** prompt. The source defaults to your hosting provider, `acquia` or `lagoon`, and to -`url` without a hosting provider. +`url` without a hosting provider. To fetch the migration dump from an FTP +or S3 store that the nightly CI run refreshes, choose it at the +**Migration database store** prompt. See [Database +storage](database-storage.mdx#multiple-databases). ### Manual setup diff --git a/.vortex/docs/content/development/provision.mdx b/.vortex/docs/content/development/provision.mdx index 4117778be9..aaae42693a 100644 --- a/.vortex/docs/content/development/provision.mdx +++ b/.vortex/docs/content/development/provision.mdx @@ -379,9 +379,9 @@ following values: Any other value stops the command with an error that lists these values. -An `ftp` or `s3` source can also be a dump store that the nightly CI run -refreshes with a sanitized dump. See [Database -storage](database-storage.mdx). +To give developers a sanitized dump without access to the source, set +`VORTEX_FETCH_DB_STORE` to an FTP or S3 store that the nightly CI run +refreshes. See [Database storage](database-storage.mdx). Each source reads its own settings, which the [Variables reference](variables.mdx) lists. The hosting sources copy the diff --git a/.vortex/docs/content/development/variables.mdx b/.vortex/docs/content/development/variables.mdx index 752dac5909..f4b1b79e11 100644 --- a/.vortex/docs/content/development/variables.mdx +++ b/.vortex/docs/content/development/variables.mdx @@ -327,6 +327,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB2_S3_BUCKET` | AWS S3 bucket name for second database fetch. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_S3_REGION` | AWS S3 region. | `ap-southeast-2` | `.env` | | `VORTEX_FETCH_DB2_SOURCE` | Second database fetch source. | `url` | `.env` | +| `VORTEX_FETCH_DB2_STORE` | Database store of the second database dump file: ftp or s3.

When set, the second database is fetched from the store instead of the source. The nightly CI job fetches from the source and pushes the dump, as fetched, to the store, next to the primary dump. Leave empty to fetch from the source. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB2_URL` | Second database dump file sourced from a URL.

HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env` | | `VORTEX_FETCH_DB_ACQUIA_APP_NAME` | Application name. Used to discover UUID. | `${VORTEX_ACQUIA_APP_NAME}` | `.vortex/tooling/src/vortex-fetch-db-acquia` | | `VORTEX_FETCH_DB_ACQUIA_BACKUP_MAX_WAIT` | Maximum time in seconds to wait for backup completion. | `600` | `.vortex/tooling/src/vortex-fetch-db-acquia` | @@ -345,7 +346,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_ENVIRONMENT` | Environment to fetch the database from.

Applies to hosting environments. Note that depending on the hosting provider, this variable may represent a branch name or an environment name. | `prod` | `.env`, `.vortex/tooling/src/vortex-fetch-db-acquia`, `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db` | | `VORTEX_FETCH_DB_FORCE` | Always override existing fetched DB dump. | `1` | `.env.local.example`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-fetch-db-container-registry` | -| `VORTEX_FETCH_DB_FRESH` | Flag to fetch a fresh copy of the database by triggering a new backup. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-acquia`, `.vortex/tooling/src/vortex-fetch-db-lagoon` | +| `VORTEX_FETCH_DB_FRESH` | Fetch a fresh database backup from the source. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-fetch-db-acquia`, `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_DB_FILE` | Database dump file name. | `db.sql` | `.vortex/tooling/src/vortex-fetch-db-ftp` | | `VORTEX_FETCH_DB_FTP_DIR` | Database dump FTP directory.

Leave empty for the root directory of the FTP account. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-ftp` | @@ -374,6 +375,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_FETCH_DB_SOURCE` | Database fetch source. | `url` | `.env`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-push-db` | | `VORTEX_FETCH_DB_SSH_FILE` | Default SSH file used if custom fingerprint is not provided. | `${HOME}/.ssh/id_rsa` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | | `VORTEX_FETCH_DB_SSH_FINGERPRINT` | SSH key fingerprint used to connect to a remote. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-lagoon` | +| `VORTEX_FETCH_DB_STORE` | Database store of the sanitized database dump file: ftp or s3.

When set, the database is fetched from the store instead of the source. The nightly CI job fetches from the source, sanitizes the database and pushes the dump to the store. The store reads the FTP or S3 variables above. Leave empty to fetch from the source. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db`, `.vortex/tooling/src/vortex-push-db` | | `VORTEX_FETCH_DB_UNZIP_PASSWORD` | Password for unzipping password-protected zip files. | `UNDEFINED` | `.vortex/tooling/src/vortex-fetch-db-url` | | `VORTEX_FETCH_DB_URL` | Database dump file sourced from a URL.

HTTP Basic Authentication credentials should be embedded into the value. | `UNDEFINED` | `.env`, `.vortex/tooling/src/vortex-fetch-db-url` | | `VORTEX_FETCH_DB_URL_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-fetch-db-url` | @@ -551,7 +553,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_CONTAINER_REGISTRY_MAP` | Comma-separated map of container services and images to push in format "service1=org/image1,service2=org/image2". | `UNDEFINED` | `.vortex/tooling/src/vortex-push-container-registry`, `.vortex/tooling/src/vortex-push-db-image` | | `VORTEX_PUSH_CONTAINER_REGISTRY_PASS` | The password to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_PASS}` | `.vortex/tooling/src/vortex-push-container-registry` | | `VORTEX_PUSH_CONTAINER_REGISTRY_USER` | The username to login into the container registry. | `${VORTEX_CONTAINER_REGISTRY_USER}` | `.vortex/tooling/src/vortex-push-container-registry` | -| `VORTEX_PUSH_DB_DESTINATIONS` | Destinations to push the database dump file to: 'ftp', 's3' or both, separated by commas or spaces.

Defaults to the fetch source when it is a dump store. Each destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for 'ftp'. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | +| `VORTEX_PUSH_DB_DESTINATIONS` | Destinations to push the database dump file to besides the database store: 'ftp', 's3' or both, separated by commas or spaces.

Each destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for 'ftp'. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db` | | `VORTEX_PUSH_DB_FILE_SUFFIX` | Suffix for the names of the pushed database dump files.

Goes before the extension, so 'db.sql' becomes 'db_gha.sql' with '_gha'. Use a different suffix in each place that pushes to the same storage. A fetch reads the suffixed name only when its remote file name includes it. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_FTP_DB_DIR` | Directory with database dump file. | `./.data` | `.vortex/tooling/src/vortex-push-db-ftp` | | `VORTEX_PUSH_DB_FTP_DB_FILE` | Database dump file name.

Defaults to the name the database is fetched and refreshed with. | `db.sql` | `.vortex/tooling/src/vortex-push-db-ftp` | @@ -573,7 +575,7 @@ The list below is automatically generated with [Shellvar](https://github.com/ale | `VORTEX_PUSH_DB_S3_STORAGE_CLASS` | S3 storage class. | `STANDARD` | `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_PUSH_DB_TIMESTAMPED_COPY` | Also push a copy of the database dump file named with a UTC timestamp.

Set to `1` to keep a history of the pushed dumps. The fetch never reads the timestamped copies. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db-ftp`, `.vortex/tooling/src/vortex-push-db-s3` | | `VORTEX_REFRESH_DB_PROCEED` | Proceed with the refresh of an indexed database.

The primary database is always refreshed. An indexed database, such as the migration database with VORTEX_DB_INDEX=`2`, is refreshed only when VORTEX_REFRESH_DB2_PROCEED is set to `1`. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | -| `VORTEX_REFRESH_DB_SOURCE` | Upstream source of the pushed database dump file. | `UNDEFINED` | `.vortex/tooling/src/vortex-push-db`, `.vortex/tooling/src/vortex-refresh-db` | +| `VORTEX_REFRESH_DB_SOURCE` | Upstream fetch source of the database dump file.

Required when the fetch source is 'container_registry' and a base image is set: the database image is built from a dump file, so it needs a dump file source such as 'url', 'ftp', 'acquia', 'lagoon' or 's3'. An indexed database uses VORTEX_REFRESH_DB2_SOURCE. | `UNDEFINED` | `.vortex/tooling/src/vortex-refresh-db` | | `VORTEX_RELEASE_VERSION_SCHEME` | Versioning scheme used for releases.

Can be one of: calver, semver, other @see https://www.vortextemplate.com/docs/releasing | `calver` | `.env` | | `VORTEX_SHOW_LOGIN` | Show one-time login link. | `UNDEFINED` | `.vortex/tooling/src/vortex-info` | | `VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING` | Disable strict host key checking in SSH. | `0` | `.vortex/tooling/src/vortex-setup-ssh` | diff --git a/.vortex/docs/content/features.mdx b/.vortex/docs/content/features.mdx index 6880fab076..bc0b7e3603 100644 --- a/.vortex/docs/content/features.mdx +++ b/.vortex/docs/content/features.mdx @@ -387,6 +387,8 @@ import { - Database sourced from the hosting provider, a URL, FTP, S3 or a container image seeded with [Database data images](https://github.com/drevops/database-data-images) + - Optional sanitized database store on FTP or S3, refreshed nightly or + on demand, so developers need no access to production ### Provisioning From e524c5724786d800f129d36a9af072a93f19e541 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 18:21:29 +1100 Subject: [PATCH 35/40] [#3340] Added the 'Database store' and 'Migration database store' installer prompts that set 'VORTEX_FETCH_DB_STORE' and keep the store connection settings. --- .../Prompts/Handlers/DatabaseFetchSource.php | 11 +- .../Prompts/Handlers/DatabaseFetchStore.php | 109 ++++++++++++++++++ .../Prompts/Handlers/MigrationFetchSource.php | 11 +- .../Prompts/Handlers/MigrationFetchStore.php | 109 ++++++++++++++++++ .../installer/src/Prompts/PromptManager.php | 13 +++ .../Handlers/BaselineHandlerProcessTest.php | 2 + .../DatabaseFetchSourceHandlerProcessTest.php | 26 +++++ ...MigrationFetchSourceHandlerProcessTest.php | 19 ++- .../AbstractHandlerDiscoveryTestCase.php | 6 + .../Handlers/AbstractHandlerTypeTest.php | 4 + ...DatabaseFetchStoreHandlerDiscoveryTest.php | 72 ++++++++++++ .../DatabaseImageHandlerDiscoveryTest.php | 8 +- ...grationFetchSourceHandlerDiscoveryTest.php | 16 ++- ...igrationFetchStoreHandlerDiscoveryTest.php | 103 +++++++++++++++++ .../MigrationHandlerDiscoveryTest.php | 7 +- .../ProvisionTypeHandlerDiscoveryTest.php | 3 +- .../tests/Unit/Schema/SchemaGeneratorTest.php | 6 + 17 files changed, 500 insertions(+), 25 deletions(-) create mode 100644 .vortex/installer/src/Prompts/Handlers/DatabaseFetchStore.php create mode 100644 .vortex/installer/src/Prompts/Handlers/MigrationFetchStore.php create mode 100644 .vortex/installer/tests/Unit/Prompts/Handlers/DatabaseFetchStoreHandlerDiscoveryTest.php create mode 100644 .vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchStoreHandlerDiscoveryTest.php diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php index baf3d8b360..d463e6fcb4 100644 --- a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchSource.php @@ -134,9 +134,12 @@ public function process(): void { self::S3, ]; + // The store reads the connection settings of the source type it names. + $store = in_array($v, DatabaseFetchStore::SOURCES, TRUE) ? ($this->responses[DatabaseFetchStore::id()] ?? NULL) : NULL; + foreach ($types as $type) { $token = 'DB_FETCH_SOURCE_' . strtoupper($type); - if ($v === $type) { + if ($v === $type || $store === $type) { File::removeTokenAsync('!' . $token); } else { @@ -150,12 +153,6 @@ public function process(): void { if ($v !== self::CONTAINER_REGISTRY && !$is_migration_in_image) { File::removeTokenAsync('DB_IN_IMAGE'); } - - // DB_PUSH wraps the upstream source of a dump store, which only an - // 'ftp' or 's3' source can be. - if (!in_array($v, [self::FTP, self::S3], TRUE)) { - File::removeTokenAsync('DB_PUSH'); - } } } diff --git a/.vortex/installer/src/Prompts/Handlers/DatabaseFetchStore.php b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchStore.php new file mode 100644 index 0000000000..a95a0295f6 --- /dev/null +++ b/.vortex/installer/src/Prompts/Handlers/DatabaseFetchStore.php @@ -0,0 +1,109 @@ + 'None, fetch from the source', + self::FTP => 'FTP server', + self::S3 => 'S3 bucket', + ]; + + // The nightly job fetches from the source and pushes to the store, so + // the 2 must differ. + if (isset($responses[DatabaseFetchSource::id()]) && is_string($responses[DatabaseFetchSource::id()])) { + unset($options[$responses[DatabaseFetchSource::id()]]); + } + + return $options; + } + + /** + * {@inheritdoc} + */ + public function dependsOn(): ?array { + return [DatabaseFetchSource::id() => self::SOURCES]; + } + + /** + * {@inheritdoc} + */ + public function shouldRun(array $responses): bool { + return isset($responses[DatabaseFetchSource::id()]) && in_array($responses[DatabaseFetchSource::id()], self::SOURCES, TRUE); + } + + /** + * {@inheritdoc} + */ + public function default(array $responses): null|string|bool|array { + return self::NONE; + } + + /** + * {@inheritdoc} + */ + public function discover(): null|string|bool|array { + $value = Env::getFromDotenv('VORTEX_FETCH_DB_STORE', $this->destinationDir); + + return in_array($value, [self::FTP, self::S3], TRUE) ? $value : NULL; + } + + /** + * {@inheritdoc} + */ + public function process(): void { + $t = $this->tmpDir; + + $v = $this->shouldRun($this->responses) && in_array($this->response, [self::FTP, self::S3], TRUE) && $this->response !== $this->responses[DatabaseFetchSource::id()] ? $this->getResponseAsString() : ''; + + Env::writeValueDotenv('VORTEX_FETCH_DB_STORE', $v, $t . '/.env'); + } + +} diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php index 4307d6847d..6cba48dffe 100644 --- a/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchSource.php @@ -135,21 +135,18 @@ public function process(): void { self::S3, ]; + // The store reads the connection settings of the source type it names. + $store = in_array($v, MigrationFetchStore::SOURCES, TRUE) ? ($this->responses[MigrationFetchStore::id()] ?? NULL) : NULL; + foreach ($types as $type) { $token = 'MIGRATION_DB_FETCH_SOURCE_' . strtoupper($type); - if ($v === $type) { + if ($v === $type || $store === $type) { File::removeTokenAsync('!' . $token); } else { File::removeTokenAsync($token); } } - - // MIGRATION_DB_PUSH wraps the upstream source of a dump store, which - // only an 'ftp' or 's3' source can be. - if (!in_array($v, [self::FTP, self::S3], TRUE)) { - File::removeTokenAsync('MIGRATION_DB_PUSH'); - } } } diff --git a/.vortex/installer/src/Prompts/Handlers/MigrationFetchStore.php b/.vortex/installer/src/Prompts/Handlers/MigrationFetchStore.php new file mode 100644 index 0000000000..c331ba0704 --- /dev/null +++ b/.vortex/installer/src/Prompts/Handlers/MigrationFetchStore.php @@ -0,0 +1,109 @@ + 'None, fetch from the source', + self::FTP => 'FTP server', + self::S3 => 'S3 bucket', + ]; + + // The nightly job fetches from the source and pushes to the store, so + // the 2 must differ. + if (isset($responses[MigrationFetchSource::id()]) && is_string($responses[MigrationFetchSource::id()])) { + unset($options[$responses[MigrationFetchSource::id()]]); + } + + return $options; + } + + /** + * {@inheritdoc} + */ + public function dependsOn(): ?array { + return [MigrationFetchSource::id() => self::SOURCES]; + } + + /** + * {@inheritdoc} + */ + public function shouldRun(array $responses): bool { + return isset($responses[Migration::id()]) && $responses[Migration::id()] === TRUE && isset($responses[MigrationFetchSource::id()]) && in_array($responses[MigrationFetchSource::id()], self::SOURCES, TRUE); + } + + /** + * {@inheritdoc} + */ + public function default(array $responses): null|string|bool|array { + return self::NONE; + } + + /** + * {@inheritdoc} + */ + public function discover(): null|string|bool|array { + $value = Env::getFromDotenv('VORTEX_FETCH_DB2_STORE', $this->destinationDir); + + return in_array($value, [self::FTP, self::S3], TRUE) ? $value : NULL; + } + + /** + * {@inheritdoc} + */ + public function process(): void { + $t = $this->tmpDir; + + $v = $this->shouldRun($this->responses) && in_array($this->response, [self::FTP, self::S3], TRUE) && $this->response !== $this->responses[MigrationFetchSource::id()] ? $this->getResponseAsString() : ''; + + Env::writeValueDotenv('VORTEX_FETCH_DB2_STORE', $v, $t . '/.env'); + } + +} diff --git a/.vortex/installer/src/Prompts/PromptManager.php b/.vortex/installer/src/Prompts/PromptManager.php index fbac3d60c1..2bbcc230a0 100644 --- a/.vortex/installer/src/Prompts/PromptManager.php +++ b/.vortex/installer/src/Prompts/PromptManager.php @@ -14,6 +14,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\CodeProvider; use DrevOps\VortexInstaller\Prompts\Handlers\CustomModules; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesAutomerge; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesProvider; @@ -29,6 +30,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\MachineName; use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationImage; use DrevOps\VortexInstaller\Prompts\Handlers\ModulePrefix; use DrevOps\VortexInstaller\Prompts\Handlers\Modules; @@ -157,6 +159,7 @@ public function runPrompts(): bool { ->add($h[DatabaseFetchSource::id()]) ->children() ->add($h[DatabaseImage::id()]) + ->add($h[DatabaseFetchStore::id()]) ->end() ->end() ->add($h[Migration::id()]) @@ -164,6 +167,7 @@ public function runPrompts(): bool { ->add($h[MigrationFetchSource::id()]) ->children() ->add($h[MigrationImage::id()]) + ->add($h[MigrationFetchStore::id()]) ->end() ->end() ->section('Notifications') @@ -255,6 +259,7 @@ protected function normalizeResponses(array $responses): array { if (isset($responses[ProvisionType::id()]) && $responses[ProvisionType::id()] === ProvisionType::PROFILE) { $responses[DatabaseFetchSource::id()] = DatabaseFetchSource::NONE; + $responses[DatabaseFetchStore::id()] = NULL; } // Handle Starter when the installer is running in update mode. @@ -414,6 +419,10 @@ public function getResponsesSummary(): array { if ($responses[DatabaseFetchSource::id()] === DatabaseFetchSource::CONTAINER_REGISTRY) { $values['Database container image'] = $responses[DatabaseImage::id()]; } + + if (isset($responses[DatabaseFetchStore::id()])) { + $values['Database store'] = $responses[DatabaseFetchStore::id()]; + } } if (isset($responses[Migration::id()])) { @@ -424,6 +433,10 @@ public function getResponsesSummary(): array { if ($responses[MigrationFetchSource::id()] === MigrationFetchSource::CONTAINER_REGISTRY && isset($responses[MigrationImage::id()])) { $values['Migration database container image'] = $responses[MigrationImage::id()]; } + + if (isset($responses[MigrationFetchStore::id()])) { + $values['Migration database store'] = $responses[MigrationFetchStore::id()]; + } } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/BaselineHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/BaselineHandlerProcessTest.php index bbf02a388b..f00200ad88 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/BaselineHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/BaselineHandlerProcessTest.php @@ -10,6 +10,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\CiProvider; use DrevOps\VortexInstaller\Prompts\Handlers\CodeProvider; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesAutomerge; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesProvider; @@ -41,6 +42,7 @@ #[CoversClass(CiProvider::class)] #[CoversClass(CodeProvider::class)] #[CoversClass(DatabaseFetchSource::class)] +#[CoversClass(DatabaseFetchStore::class)] #[CoversClass(DatabaseImage::class)] #[CoversClass(DependencyUpdatesAutomerge::class)] #[CoversClass(DependencyUpdatesProvider::class)] diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php index 9ca89d1241..67b5b8847b 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/DatabaseFetchSourceHandlerProcessTest.php @@ -6,10 +6,12 @@ use DrevOps\VortexInstaller\Prompts\Handlers\AiCodeInstructions; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use PHPUnit\Framework\Attributes\CoversClass; #[CoversClass(DatabaseFetchSource::class)] +#[CoversClass(DatabaseFetchStore::class)] #[CoversClass(DatabaseImage::class)] class DatabaseFetchSourceHandlerProcessTest extends AbstractHandlerProcessTestCase { @@ -36,6 +38,30 @@ public static function dataProviderHandlerProcess(): \Iterator { yield 'db_fetch_source_s3' => [ static::cw(fn(AbstractHandlerProcessTestCase $test): string => $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::S3), ]; + yield 'db_fetch_store_ftp' => [ + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::LAGOON; + $test->prompts[DatabaseFetchStore::id()] = DatabaseFetchStore::FTP; + }), + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_SOURCE=lagoon'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_STORE=ftp'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_FTP_HOST='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_S3_BUCKET'); + }), + ]; + yield 'db_fetch_store_s3' => [ + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->prompts[DatabaseFetchSource::id()] = DatabaseFetchSource::URL; + $test->prompts[DatabaseFetchStore::id()] = DatabaseFetchStore::S3; + }), + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_SOURCE=url'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_STORE=s3'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_S3_BUCKET='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB_FTP_HOST'); + }), + ]; } } diff --git a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php index 2ae12c8fdd..ad0901f163 100644 --- a/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php +++ b/.vortex/installer/tests/Functional/Prompts/Handlers/MigrationFetchSourceHandlerProcessTest.php @@ -6,10 +6,12 @@ use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationImage; use PHPUnit\Framework\Attributes\CoversClass; #[CoversClass(MigrationFetchSource::class)] +#[CoversClass(MigrationFetchStore::class)] class MigrationFetchSourceHandlerProcessTest extends AbstractHandlerProcessTestCase { public static function dataProviderHandlerProcess(): \Iterator { @@ -27,9 +29,22 @@ public static function dataProviderHandlerProcess(): \Iterator { $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-refresh-db'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_IMAGE_BASE'); $test->assertFileNotContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'Deploy DB image'); - // Any migration source can push to its own destinations. $test->assertFileContainsString(static::$sut . '/.github/workflows/build-test-deploy.yml', 'VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db'); - $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_REFRESH_DB2_SOURCE=url'); + $test->assertFileContainsString(static::$sut . '/.env', "VORTEX_FETCH_DB2_STORE=\n"); + }), + ]; + yield 'migration_fetch_store_ftp' => [ + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->prompts[Migration::id()] = TRUE; + $test->prompts[MigrationFetchSource::id()] = MigrationFetchSource::URL; + $test->prompts[MigrationFetchStore::id()] = MigrationFetchStore::FTP; + }), + static::cw(function (AbstractHandlerProcessTestCase $test): void { + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_SOURCE=url'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_STORE=ftp'); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_URL='); + $test->assertFileContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_FTP_HOST='); + $test->assertFileNotContainsString(static::$sut . '/.env', 'VORTEX_FETCH_DB2_S3_BUCKET'); }), ]; yield 'migration_fetch_source_ftp' => [ diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerDiscoveryTestCase.php b/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerDiscoveryTestCase.php index f0ddf7bde2..a77df32196 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerDiscoveryTestCase.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerDiscoveryTestCase.php @@ -14,6 +14,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\CodeProvider; use DrevOps\VortexInstaller\Prompts\Handlers\CustomModules; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesAutomerge; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesProvider; @@ -28,6 +29,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\MachineName; use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationImage; use DrevOps\VortexInstaller\Prompts\Handlers\ModulePrefix; use DrevOps\VortexInstaller\Prompts\Handlers\Modules; @@ -159,9 +161,11 @@ protected static function getExpectedDefaults(): array { ProvisionType::id() => ProvisionType::DATABASE, DatabaseFetchSource::id() => DatabaseFetchSource::URL, DatabaseImage::id() => NULL, + DatabaseFetchStore::id() => DatabaseFetchStore::NONE, Migration::id() => FALSE, MigrationFetchSource::id() => NULL, MigrationImage::id() => NULL, + MigrationFetchStore::id() => NULL, CiProvider::id() => CiProvider::GITHUB_ACTIONS, VisualRegression::id() => FALSE, Lighthouse::id() => FALSE, @@ -246,9 +250,11 @@ public static function defaultTuiAnswers(): array { ProvisionType::id() => static::TUI_DEFAULT, DatabaseFetchSource::id() => static::TUI_DEFAULT, DatabaseImage::id() => static::TUI_SKIP, + DatabaseFetchStore::id() => static::TUI_DEFAULT, Migration::id() => static::TUI_DEFAULT, MigrationFetchSource::id() => static::TUI_SKIP, MigrationImage::id() => static::TUI_SKIP, + MigrationFetchStore::id() => static::TUI_SKIP, NotificationChannels::id() => static::TUI_DEFAULT, CiProvider::id() => static::TUI_DEFAULT, VisualRegression::id() => static::TUI_DEFAULT, diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerTypeTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerTypeTest.php index 735911c6fb..7a3b19e66a 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerTypeTest.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/AbstractHandlerTypeTest.php @@ -13,6 +13,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\CodeProvider; use DrevOps\VortexInstaller\Prompts\Handlers\CustomModules; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesAutomerge; use DrevOps\VortexInstaller\Prompts\Handlers\DependencyUpdatesProvider; @@ -27,6 +28,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\MachineName; use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationImage; use DrevOps\VortexInstaller\Prompts\Handlers\ModulePrefix; use DrevOps\VortexInstaller\Prompts\Handlers\Modules; @@ -99,7 +101,9 @@ public static function dataProviderTypeInference(): \Iterator { yield 'hosting_provider' => [HostingProvider::id(), PromptType::Select]; yield 'provision_type' => [ProvisionType::id(), PromptType::Select]; yield 'database_fetch_source' => [DatabaseFetchSource::id(), PromptType::Select]; + yield 'database_fetch_store' => [DatabaseFetchStore::id(), PromptType::Select]; yield 'migration_fetch_source' => [MigrationFetchSource::id(), PromptType::Select]; + yield 'migration_fetch_store' => [MigrationFetchStore::id(), PromptType::Select]; yield 'ci_provider' => [CiProvider::id(), PromptType::Select]; yield 'dependency_updates_provider' => [DependencyUpdatesProvider::id(), PromptType::Select]; yield 'code_coverage_provider' => [CodeCoverageProvider::id(), PromptType::Select]; diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseFetchStoreHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseFetchStoreHandlerDiscoveryTest.php new file mode 100644 index 0000000000..eacfbab1e6 --- /dev/null +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseFetchStoreHandlerDiscoveryTest.php @@ -0,0 +1,72 @@ + [ + [DatabaseFetchStore::id() => self::KEY_ENTER], + [DatabaseFetchStore::id() => DatabaseFetchStore::NONE] + $expected_defaults, + ]; + yield 'database fetch store - prompt - ftp' => [ + [DatabaseFetchStore::id() => self::KEY_DOWN . self::KEY_ENTER], + [DatabaseFetchStore::id() => DatabaseFetchStore::FTP] + $expected_defaults, + ]; + yield 'database fetch store - prompt - s3' => [ + [DatabaseFetchStore::id() => self::KEY_DOWN . self::KEY_DOWN . self::KEY_ENTER], + [DatabaseFetchStore::id() => DatabaseFetchStore::S3] + $expected_defaults, + ]; + yield 'database fetch store - prompt - source excluded' => [ + [ + DatabaseFetchSource::id() => self::KEY_DOWN . self::KEY_ENTER, + DatabaseFetchStore::id() => self::KEY_DOWN . self::KEY_ENTER, + ], + [DatabaseFetchSource::id() => DatabaseFetchSource::FTP, DatabaseFetchStore::id() => DatabaseFetchStore::S3] + $expected_defaults, + ]; + yield 'database fetch store - skipped - source none' => [ + [ + DatabaseFetchSource::id() => str_repeat(self::KEY_DOWN, 6) . self::KEY_ENTER, + DatabaseFetchStore::id() => static::TUI_SKIP, + ], + [DatabaseFetchSource::id() => DatabaseFetchSource::NONE, DatabaseFetchStore::id() => NULL] + $expected_defaults, + ]; + yield 'database fetch store - discovery - ftp' => [ + [], + [DatabaseFetchStore::id() => DatabaseFetchStore::FTP] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB_STORE', DatabaseFetchStore::FTP); + }, + ]; + yield 'database fetch store - discovery - s3' => [ + [], + [DatabaseFetchStore::id() => DatabaseFetchStore::S3] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB_STORE', DatabaseFetchStore::S3); + }, + ]; + yield 'database fetch store - discovery - empty' => [ + [], + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB_STORE', ''); + }, + ]; + yield 'database fetch store - discovery - invalid' => [ + [], + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB_STORE', 'invalid_store'); + }, + ]; + } + +} diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseImageHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseImageHandlerDiscoveryTest.php index 2eaf11b04d..9f54dbffef 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseImageHandlerDiscoveryTest.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/DatabaseImageHandlerDiscoveryTest.php @@ -5,6 +5,7 @@ namespace DrevOps\VortexInstaller\Tests\Unit\Prompts\Handlers; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Utils\Config; use PHPUnit\Framework\Attributes\CoversClass; @@ -18,13 +19,15 @@ public static function dataProviderRunPrompts(): \Iterator { [ DatabaseFetchSource::id() => self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_ENTER, DatabaseImage::id() => 'myregistry/myimage:mytag', + DatabaseFetchStore::id() => static::TUI_SKIP, ], - [DatabaseFetchSource::id() => DatabaseFetchSource::CONTAINER_REGISTRY, DatabaseImage::id() => 'myregistry/myimage:mytag'] + $expected_defaults, + [DatabaseFetchSource::id() => DatabaseFetchSource::CONTAINER_REGISTRY, DatabaseImage::id() => 'myregistry/myimage:mytag', DatabaseFetchStore::id() => NULL] + $expected_defaults, ]; yield 'database image - invalid' => [ [ DatabaseFetchSource::id() => self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_ENTER, DatabaseImage::id() => 'myregistry:myimage:mytag', + DatabaseFetchStore::id() => static::TUI_SKIP, ], 'Please enter a valid container image name with an optional tag.', ]; @@ -32,6 +35,7 @@ public static function dataProviderRunPrompts(): \Iterator { [ DatabaseFetchSource::id() => self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_ENTER, DatabaseImage::id() => 'MyRegistry/MyImage:mytag', + DatabaseFetchStore::id() => static::TUI_SKIP, ], 'Please enter a valid container image name with an optional tag.', ]; @@ -39,10 +43,12 @@ public static function dataProviderRunPrompts(): \Iterator { [ DatabaseFetchSource::id() => self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_DOWN . self::KEY_ENTER, DatabaseImage::id() => static::TUI_DEFAULT, + DatabaseFetchStore::id() => static::TUI_SKIP, ], [ DatabaseFetchSource::id() => DatabaseFetchSource::CONTAINER_REGISTRY, DatabaseImage::id() => 'discovered_owner/discovered_image:tag', + DatabaseFetchStore::id() => NULL, ] + $expected_defaults, function (AbstractHandlerDiscoveryTestCase $test, Config $config): void { $test->stubDotenvValue('VORTEX_DB_IMAGE', 'discovered_owner/discovered_image:tag'); diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchSourceHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchSourceHandlerDiscoveryTest.php index 2e800ecf29..53888550e6 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchSourceHandlerDiscoveryTest.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchSourceHandlerDiscoveryTest.php @@ -6,6 +6,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use PHPUnit\Framework\Attributes\CoversClass; #[CoversClass(MigrationFetchSource::class)] @@ -17,22 +18,25 @@ public static function dataProviderRunPrompts(): \Iterator { [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, ]; yield 'migration download source - prompt - ftp' => [ [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_DOWN . self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::FTP] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::FTP, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, ]; yield 'migration download source - discovery' => [ [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::FTP] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::FTP, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, function (AbstractHandlerDiscoveryTestCase $test): void { $test->stubDotenvValue('VORTEX_FETCH_DB2_SOURCE', MigrationFetchSource::FTP); }, @@ -41,8 +45,9 @@ function (AbstractHandlerDiscoveryTestCase $test): void { [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::S3] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::S3, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, function (AbstractHandlerDiscoveryTestCase $test): void { $test->stubDotenvValue('VORTEX_FETCH_DB2_SOURCE', MigrationFetchSource::S3); }, @@ -51,8 +56,9 @@ function (AbstractHandlerDiscoveryTestCase $test): void { [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, function (AbstractHandlerDiscoveryTestCase $test): void { $test->stubDotenvValue('VORTEX_FETCH_DB2_SOURCE', 'invalid_source'); }, diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchStoreHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchStoreHandlerDiscoveryTest.php new file mode 100644 index 0000000000..9dedc86206 --- /dev/null +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationFetchStoreHandlerDiscoveryTest.php @@ -0,0 +1,103 @@ + [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, + ]; + yield 'migration fetch store - prompt - ftp' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_DOWN . self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::FTP] + $expected_defaults, + ]; + yield 'migration fetch store - prompt - source excluded' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_DOWN . self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_DOWN . self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::FTP, MigrationFetchStore::id() => MigrationFetchStore::S3] + $expected_defaults, + ]; + yield 'migration fetch store - skipped - container registry' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => str_repeat(self::KEY_DOWN, 4) . self::KEY_ENTER, + MigrationImage::id() => static::TUI_DEFAULT, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::CONTAINER_REGISTRY, MigrationImage::id() => 'myprojectorg/myproject-data-migration:latest'] + $expected_defaults, + ]; + yield 'migration fetch store - skipped - migration disabled' => [ + [], + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB2_STORE', MigrationFetchStore::FTP); + }, + ]; + yield 'migration fetch store - discovery - ftp' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::FTP] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB2_STORE', MigrationFetchStore::FTP); + }, + ]; + yield 'migration fetch store - discovery - s3' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::S3] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB2_STORE', MigrationFetchStore::S3); + }, + ]; + yield 'migration fetch store - discovery - empty' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB2_STORE', ''); + }, + ]; + yield 'migration fetch store - discovery - invalid' => [ + [ + Migration::id() => self::KEY_LEFT . self::KEY_ENTER, + MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, + ], + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, + function (AbstractHandlerDiscoveryTestCase $test): void { + $test->stubDotenvValue('VORTEX_FETCH_DB2_STORE', 'invalid_store'); + }, + ]; + } + +} diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationHandlerDiscoveryTest.php index ca38df3687..1f8a7b5d0e 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationHandlerDiscoveryTest.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/MigrationHandlerDiscoveryTest.php @@ -6,6 +6,7 @@ use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\Services; use DrevOps\VortexInstaller\Utils\Config; use DrevOps\VortexInstaller\Utils\File; @@ -26,14 +27,16 @@ public static function dataProviderRunPrompts(): \Iterator { [ Migration::id() => self::KEY_LEFT . self::KEY_ENTER, MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL] + $expected_defaults, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_defaults, ]; yield 'migration - discovery - database2 exists' => [ [ MigrationFetchSource::id() => self::KEY_ENTER, + MigrationFetchStore::id() => self::KEY_ENTER, ], - [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL] + $expected_installed, + [Migration::id() => TRUE, MigrationFetchSource::id() => MigrationFetchSource::URL, MigrationFetchStore::id() => MigrationFetchStore::NONE] + $expected_installed, function (AbstractHandlerDiscoveryTestCase $test, Config $config): void { $test->stubVortexProject($config); File::dump(static::$sut . '/docker-compose.yml', Yaml::dump(['services' => ['database2' => [], Services::CLAMAV => [], Services::REDIS => [], Services::SOLR => []]])); diff --git a/.vortex/installer/tests/Unit/Prompts/Handlers/ProvisionTypeHandlerDiscoveryTest.php b/.vortex/installer/tests/Unit/Prompts/Handlers/ProvisionTypeHandlerDiscoveryTest.php index 699d1c462e..108ecb95e7 100644 --- a/.vortex/installer/tests/Unit/Prompts/Handlers/ProvisionTypeHandlerDiscoveryTest.php +++ b/.vortex/installer/tests/Unit/Prompts/Handlers/ProvisionTypeHandlerDiscoveryTest.php @@ -5,6 +5,7 @@ namespace DrevOps\VortexInstaller\Tests\Unit\Prompts\Handlers; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\ProvisionType; use DrevOps\VortexInstaller\Utils\Config; use PHPUnit\Framework\Attributes\CoversClass; @@ -27,7 +28,7 @@ function (AbstractHandlerDiscoveryTestCase $test, Config $config): void { ]; yield 'provision type - discovery - profile' => [ [], - [ProvisionType::id() => ProvisionType::PROFILE, DatabaseFetchSource::id() => DatabaseFetchSource::NONE] + $expected_defaults, + [ProvisionType::id() => ProvisionType::PROFILE, DatabaseFetchSource::id() => DatabaseFetchSource::NONE, DatabaseFetchStore::id() => NULL] + $expected_defaults, function (AbstractHandlerDiscoveryTestCase $test, Config $config): void { $test->stubDotenvValue('VORTEX_PROVISION_TYPE', ProvisionType::PROFILE); }, diff --git a/.vortex/installer/tests/Unit/Schema/SchemaGeneratorTest.php b/.vortex/installer/tests/Unit/Schema/SchemaGeneratorTest.php index 666a492abe..d252aa6597 100644 --- a/.vortex/installer/tests/Unit/Schema/SchemaGeneratorTest.php +++ b/.vortex/installer/tests/Unit/Schema/SchemaGeneratorTest.php @@ -8,11 +8,13 @@ use DrevOps\VortexInstaller\Prompts\Handlers\AiCodeReview; use DrevOps\VortexInstaller\Prompts\Handlers\CiProvider; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\DatabaseImage; use DrevOps\VortexInstaller\Prompts\Handlers\HostingProjectName; use DrevOps\VortexInstaller\Prompts\Handlers\HostingProvider; use DrevOps\VortexInstaller\Prompts\Handlers\Migration; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchSource; +use DrevOps\VortexInstaller\Prompts\Handlers\MigrationFetchStore; use DrevOps\VortexInstaller\Prompts\Handlers\MigrationImage; use DrevOps\VortexInstaller\Prompts\Handlers\Name; use DrevOps\VortexInstaller\Prompts\Handlers\ProfileCustom; @@ -78,6 +80,8 @@ public static function dataProviderPromptTypes(): \Iterator { yield 'ci_provider' => [CiProvider::id(), 'select']; yield 'migration' => [Migration::id(), 'confirm']; yield 'database_image' => [DatabaseImage::id(), 'text']; + yield 'database_fetch_store' => [DatabaseFetchStore::id(), 'select']; + yield 'migration_fetch_store' => [MigrationFetchStore::id(), 'select']; yield 'ai_code_instructions' => [AiCodeInstructions::id(), 'confirm']; yield 'ai_code_review' => [AiCodeReview::id(), 'confirm']; } @@ -118,8 +122,10 @@ public static function dataProviderDependsOnFormat(): \Iterator { yield 'theme_custom' => [ThemeCustom::id()]; yield 'database_fetch_source' => [DatabaseFetchSource::id()]; yield 'database_image' => [DatabaseImage::id()]; + yield 'database_fetch_store' => [DatabaseFetchStore::id()]; yield 'migration_fetch_source' => [MigrationFetchSource::id()]; yield 'migration_image' => [MigrationImage::id()]; + yield 'migration_fetch_store' => [MigrationFetchStore::id()]; } public function testUtilityHandlersExcluded(): void { From 7f51fd4b4d963718a73c4b22094b7da964af18d7 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 18:21:35 +1100 Subject: [PATCH 36/40] Updated snapshots. --- .../Fixtures/handler_process/_baseline/.env | 17 +- .../.github/workflows/build-test-deploy.yml | 15 +- .../ciprovider_circleci/.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../code_provider_other/.circleci/config.yml | 18 +- .../db_fetch_source_acquia/.env | 8 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../db_fetch_source_container_registry/.env | 8 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../handler_process/db_fetch_source_ftp/.env | 19 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../db_fetch_source_lagoon/.env | 10 +- .../handler_process/db_fetch_source_s3/.env | 23 +-- .../.github/workflows/build-test-deploy.yml | 2 +- .../handler_process/db_fetch_store_ftp/.env | 65 ++++++ .../db_fetch_store_ftp/.env.local.example | 16 ++ .../.github/workflows/build-test-deploy.yml | 9 + .../handler_process/db_fetch_store_s3/.env | 24 +++ .../db_fetch_store_s3/.env.local.example | 11 ++ .../.github/workflows/build-test-deploy.yml | 9 + .../.circleci/config.yml | 18 +- .../deploy_types_all_circleci/.env | 2 +- .../handler_process/deploy_types_all_gha/.env | 2 +- .../deploy_types_artifact/.env | 2 +- .../handler_process/deploy_types_lagoon/.env | 2 +- .../.circleci/config.yml | 18 +- .../deploy_types_none_circleci/.env | 2 +- .../deploy_types_none_gha/.env | 2 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.circleci/config.yml | 18 +- .../handler_process/hosting_acquia/.env | 14 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../handler_process/hosting_lagoon/.env | 14 +- .../hosting_project_name___acquia/.env | 14 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../hosting_project_name___lagoon/.env | 14 +- .../.circleci/config.yml | 18 +- .../migration_disabled_lagoon/.env | 14 +- .../handler_process/migration_enabled/.env | 15 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.circleci/config.yml | 18 +- .../migration_enabled_circleci/.env | 15 +- .../migration_enabled_lagoon/.env | 25 ++- .../.github/workflows/build-test-deploy.yml | 6 +- .../migration_fetch_source_acquia/.env | 13 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.env | 13 +- .../.github/workflows/build-test-deploy.yml | 10 +- .../migration_fetch_source_ftp/.env | 16 +- .../.github/workflows/build-test-deploy.yml | 8 +- .../migration_fetch_source_lagoon/.env | 13 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../migration_fetch_source_s3/.env | 16 +- .../.github/workflows/build-test-deploy.yml | 8 +- .../migration_fetch_source_url/.env | 15 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../migration_fetch_store_ftp/.ahoy.yml | 51 +++++ .../migration_fetch_store_ftp/.env | 65 ++++++ .../.github/workflows/build-test-deploy.yml | 46 +++++ .../migration_fetch_store_ftp/composer.json | 9 + .../docker-compose.yml | 52 +++++ .../scripts/provision-20-migration.sh | 187 ++++++++++++++++++ .../phpunit/Drupal/DatabaseSettingsTest.php | 110 +++++++++++ .../Drupal/EnvironmentSettingsTest.php | 14 ++ .../tests/phpunit/Drupal/SettingsTestCase.php | 8 + .../migrations/ys_migrate_categories.yml | 29 +++ .../custom/ys_migrate/ys_migrate.info.yml | 9 + .../web/sites/default/settings.migration.php | 21 ++ .../web/sites/default/settings.php | 15 ++ .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../tests/Fixtures/handler_process/names/.env | 4 +- .../notification_channels_all/.env | 4 +- .../notification_channels_github_only/.env | 2 +- .../notification_channels_jira_only/.env | 2 +- .../notification_channels_newrelic_only/.env | 2 +- .../notification_channels_none/.env | 2 +- .../notification_channels_slack_only/.env | 2 +- .../notification_channels_webhook_only/.env | 2 +- .../provision_database_lagoon/.env | 14 +- .../handler_process/provision_profile/.env | 21 +- .../.github/workflows/build-test-deploy.yml | 27 +-- .../starter_drupal_cms_profile/.env | 2 +- .../starter_drupal_profile/.env | 2 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../theme_custom_non_vortex/.env | 4 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../timezone_circleci/.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 10 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 10 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 12 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../tools_no_dclint_migration/.env | 15 +- .../.github/workflows/build-test-deploy.yml | 8 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 4 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 6 +- .../.github/workflows/build-test-deploy.yml | 2 +- .../.circleci/config.yml | 18 +- .../.github/workflows/build-test-deploy.yml | 14 +- .../handler_process/version_scheme_other/.env | 2 +- .../version_scheme_semver/.env | 2 +- 133 files changed, 1536 insertions(+), 322 deletions(-) create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env.local.example create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env.local.example create mode 100644 .vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.ahoy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.env create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.github/workflows/build-test-deploy.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/composer.json create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/docker-compose.yml create mode 100755 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/scripts/provision-20-migration.sh create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/DatabaseSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/EnvironmentSettingsTest.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/SettingsTestCase.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/ys_migrate.info.yml create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.migration.php create mode 100644 .vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.php diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env index 8a3c374718..ee6b52c272 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.env @@ -187,11 +187,18 @@ VORTEX_FETCH_DB_SOURCE=url # HTTP Basic Authentication credentials should be embedded into the value. VORTEX_FETCH_DB_URL= -# Destinations the nightly CI job pushes the sanitized database dump to: -# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source -# when it is a dump store. Each destination reads its own variables, such as -# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 -# source. +# Database store of the sanitized database dump file: ftp or s3. +# +# When set, the database is fetched from the store instead of the source. The +# nightly CI job fetches from the source, sanitizes the database and pushes +# the dump to the store. The store reads the FTP or S3 variables above. Leave +# empty to fetch from the source. +VORTEX_FETCH_DB_STORE= + +# Destinations the nightly CI job pushes the sanitized database dump to +# besides the database store: ftp, s3 or both, such as "ftp,s3". Each +# destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for ftp. +# The database is always fetched from 1 place. # VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 # Also push a copy of each dump named with the time of the push, such as diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index cf49a27441..6c5c9f847b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -52,6 +52,11 @@ on: description: 'Enable terminal session for CI jobs' required: false default: false + refresh_db: + type: boolean + description: 'Refresh the database only' + required: false + default: false schedule: - cron: '0 18 * * *' @@ -72,7 +77,7 @@ jobs: lint: runs-on: ubuntu-latest - if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} + if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} container: # https://hub.docker.com/r/drevops/ci-runner @@ -236,12 +241,12 @@ jobs: run: ./scripts/vortex-tooling.sh - name: Adjust variables for a scheduled run - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' || inputs.refresh_db run: | { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" @@ -301,7 +306,7 @@ jobs: build: runs-on: ubuntu-latest needs: database - if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} + if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} permissions: contents: read @@ -588,7 +593,7 @@ jobs: deploy: runs-on: ubuntu-latest needs: [build, lint] - if: ${{ !cancelled() && (inputs.deploy_target || (success() && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} + if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} permissions: contents: read diff --git a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/ciprovider_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml index fc8bc445cb..929738c015 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -501,6 +501,17 @@ +@@ -506,6 +506,17 @@ hide_and_recreate: true diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml index 46bc8293e1..aaef53ce07 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_coverage_provider_codecov_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -596,6 +603,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -671,3 +680,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/code_provider_other/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env index 9cdae39af9..693bfe330d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.env @@ -10,10 +10,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -265,17 +260,3 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -272,17 +267,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml index a4f3384e34..bb8d3535a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -271,6 +271,9 @@ +@@ -276,6 +276,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env index c9b8a0278f..56cbec1399 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.env @@ -19,10 +19,10 @@ +# VORTEX_FETCH_DB_URL for url. Add them, and keep secrets in CI. +# VORTEX_REFRESH_DB_SOURCE=url + - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -265,17 +270,3 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -272,17 +277,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 748874cfc7..684d576b85 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -241,6 +241,10 @@ +@@ -246,6 +246,10 @@ { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" -@@ -276,6 +280,15 @@ +@@ -281,6 +285,15 @@ run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi ./vendor/bin/vortex-refresh-db diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env index 36f8efc921..2fe7dfcc73 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -180,13 +180,33 @@ +@@ -180,13 +180,24 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -24,19 +24,10 @@ +# Database dump FTP file name. +VORTEX_FETCH_DB_FTP_FILE=db.sql + -+# Upstream source of the database dump file, such as url, acquia or lagoon. -+# -+# Uncomment to make the database fetch source a dump store: the nightly CI -+# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source as the latest dump. The fetch script of this source reads its -+# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep -+# secrets in CI. -+# VORTEX_REFRESH_DB_SOURCE=url -+ - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -265,17 +285,3 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -272,17 +283,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml index 6ed86e851c..553c05f08b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -191,6 +191,8 @@ +@@ -196,6 +196,8 @@ VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env index 9fba49ca12..659f31b092 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_lagoon/.env @@ -10,10 +10,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,7 +198,7 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,7 +205,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -22,7 +22,7 @@ ################################################################################ # FILES SOURCE # -@@ -265,17 +260,3 @@ +@@ -272,17 +267,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env index 591f9ffe6d..e6758f9ccd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -180,12 +180,24 @@ +@@ -180,13 +180,16 @@ VORTEX_DB_FILE=db.sql # Database fetch source. @@ -6,28 +6,21 @@ +VORTEX_FETCH_DB_SOURCE=s3 -# Database dump file sourced from a URL. +-# +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= +# Database dump file sourced from S3. -+ + +# AWS S3 bucket name for database fetch. +VORTEX_FETCH_DB_S3_BUCKET= + +# AWS S3 region. +VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 + -+# Upstream source of the database dump file, such as url, acquia or lagoon. + # Database store of the sanitized database dump file: ftp or s3. # --# HTTP Basic Authentication credentials should be embedded into the value. --VORTEX_FETCH_DB_URL= -+# Uncomment to make the database fetch source a dump store: the nightly CI -+# job fetches the dump from this source, sanitizes it and pushes it to the -+# fetch source as the latest dump. The fetch script of this source reads its -+# own variables, such as VORTEX_FETCH_DB_URL for url. Add them, and keep -+# secrets in CI. -+# VORTEX_REFRESH_DB_SOURCE=url - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source -@@ -265,17 +277,3 @@ + # When set, the database is fetched from the store instead of the source. The +@@ -272,17 +275,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml index a9c60fbab3..ca27e284e8 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -191,6 +191,8 @@ +@@ -196,6 +196,8 @@ VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env new file mode 100644 index 0000000000..426ba38eb0 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env @@ -0,0 +1,65 @@ +@@ -180,13 +180,24 @@ + VORTEX_DB_FILE=db.sql + + # Database fetch source. +-VORTEX_FETCH_DB_SOURCE=url ++VORTEX_FETCH_DB_SOURCE=lagoon + +-# Database dump file sourced from a URL. ++# Database dump file sourced from FTP. ++ ++# Database dump FTP host. ++VORTEX_FETCH_DB_FTP_HOST= ++ ++# Database dump FTP port. ++VORTEX_FETCH_DB_FTP_PORT=21 ++ ++# Database dump FTP directory. + # +-# HTTP Basic Authentication credentials should be embedded into the value. +-VORTEX_FETCH_DB_URL= ++# Leave empty for the root directory of the FTP account. ++VORTEX_FETCH_DB_FTP_DIR= + ++# Database dump FTP file name. ++VORTEX_FETCH_DB_FTP_FILE=db.sql ++ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -193,7 +204,7 @@ + # nightly CI job fetches from the source, sanitizes the database and pushes + # the dump to the store. The store reads the FTP or S3 variables above. Leave + # empty to fetch from the source. +-VORTEX_FETCH_DB_STORE= ++VORTEX_FETCH_DB_STORE=ftp + + # Destinations the nightly CI job pushes the sanitized database dump to + # besides the database store: ftp, s3 or both, such as "ftp,s3". Each +@@ -210,7 +221,7 @@ + # Applies to hosting environments. + # Note that depending on the hosting provider, this variable may represent + # a branch name or an environment name. +-VORTEX_FETCH_DB_ENVIRONMENT=prod ++VORTEX_FETCH_DB_ENVIRONMENT=main + + ################################################################################ + # FILES SOURCE # +@@ -272,17 +283,3 @@ + # with optional names in the format "email|name". + # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" + VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" +- +-################################################################################ +-# DEMO # +-################################################################################ +-# # +-# Override project-specific values for demonstration purposes. # +-# Used to showcase Vortex without asking users to perform additional steps. # +-# # +-# Remove this section after completing database fetch integration. # +-# # +-################################################################################ +- +-# URL of the database used for demonstration with URL database fetch type. +-VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env.local.example new file mode 100644 index 0000000000..9d6e82be46 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.env.local.example @@ -0,0 +1,16 @@ +@@ -32,3 +32,15 @@ + + # Always override existing fetched DB dump. + VORTEX_FETCH_DB_FORCE=1 ++ ++# Database dump file sourced from FTP. ++ ++# Database dump FTP user. ++VORTEX_FETCH_DB_FTP_USER= ++# Database dump FTP password. ++VORTEX_FETCH_DB_FTP_PASS= ++ ++# Database dump file sourced from Lagoon. ++ ++# SSH file used to fetch the database dump from Lagoon. Defaults to "${HOME}/.ssh/id_rsa". ++# VORTEX_FETCH_DB_SSH_FILE= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..553c05f08b --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_ftp/.github/workflows/build-test-deploy.yml @@ -0,0 +1,9 @@ +@@ -196,6 +196,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_FTP_USER: ${{ secrets.VORTEX_FETCH_DB_FTP_USER }} ++ VORTEX_FETCH_DB_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env new file mode 100644 index 0000000000..5607339594 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env @@ -0,0 +1,24 @@ +@@ -187,6 +187,14 @@ + # HTTP Basic Authentication credentials should be embedded into the value. + VORTEX_FETCH_DB_URL= + ++# Database dump file sourced from S3. ++ ++# AWS S3 bucket name for database fetch. ++VORTEX_FETCH_DB_S3_BUCKET= ++ ++# AWS S3 region. ++VORTEX_FETCH_DB_S3_REGION=ap-southeast-2 ++ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -193,7 +201,7 @@ + # nightly CI job fetches from the source, sanitizes the database and pushes + # the dump to the store. The store reads the FTP or S3 variables above. Leave + # empty to fetch from the source. +-VORTEX_FETCH_DB_STORE= ++VORTEX_FETCH_DB_STORE=s3 + + # Destinations the nightly CI job pushes the sanitized database dump to + # besides the database store: ftp, s3 or both, such as "ftp,s3". Each diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env.local.example b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env.local.example new file mode 100644 index 0000000000..624f93c6d0 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.env.local.example @@ -0,0 +1,11 @@ +@@ -32,3 +32,10 @@ + + # Always override existing fetched DB dump. + VORTEX_FETCH_DB_FORCE=1 ++ ++# Database dump file sourced from S3. ++ ++# AWS access key for S3 database fetch. ++VORTEX_FETCH_DB_S3_ACCESS_KEY= ++# AWS secret key for S3 database fetch. ++VORTEX_FETCH_DB_S3_SECRET_KEY= diff --git a/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..ca27e284e8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/db_fetch_store_s3/.github/workflows/build-test-deploy.yml @@ -0,0 +1,9 @@ +@@ -196,6 +196,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB_S3_ACCESS_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_ACCESS_KEY }} ++ VORTEX_FETCH_DB_S3_SECRET_KEY: ${{ secrets.VORTEX_FETCH_DB_S3_SECRET_KEY }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env index fff488a98d..eccdba8c1e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_circleci/.env @@ -1,4 +1,4 @@ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env index fff488a98d..eccdba8c1e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_all_gha/.env @@ -1,4 +1,4 @@ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env index aec0c46a96..ecbc8fb672 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_artifact/.env @@ -1,4 +1,4 @@ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env index 65058d435a..4ff3adb871 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_lagoon/.env @@ -1,4 +1,4 @@ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml index ed97a2c0fa..6f87f4dfd4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -522,6 +529,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -558,3 +567,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env index 64c4894783..aef76967d9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_circleci/.env @@ -1,4 +1,4 @@ -@@ -233,14 +233,6 @@ +@@ -240,14 +240,6 @@ VORTEX_RELEASE_VERSION_SCHEME=calver ################################################################################ diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env index 64c4894783..aef76967d9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.env @@ -1,4 +1,4 @@ -@@ -233,14 +233,6 @@ +@@ -240,14 +240,6 @@ VORTEX_RELEASE_VERSION_SCHEME=calver ################################################################################ diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index 7b5594d096..fcb094be56 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -584,104 +584,3 @@ +@@ -589,104 +589,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true @@ -6,7 +6,7 @@ - deploy: - runs-on: ubuntu-latest - needs: [build, lint] -- if: ${{ !cancelled() && (inputs.deploy_target || (success() && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} +- if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} - - permissions: - contents: read diff --git a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deps_updates_provider_ci_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env index 9cc68201a1..2e71b9227a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.env @@ -33,10 +33,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -205,6 +207,9 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -212,6 +214,9 @@ # a branch name or an environment name. VORTEX_FETCH_DB_ENVIRONMENT=prod @@ -46,7 +46,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -213,15 +218,8 @@ +@@ -220,15 +225,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -63,7 +63,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +236,7 @@ +@@ -245,7 +243,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -72,7 +72,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +263,3 @@ +@@ -272,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml index a4f3384e34..bb8d3535a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -271,6 +271,9 @@ +@@ -276,6 +276,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env index a1d35bca50..cfd0cd8e9f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_lagoon/.env @@ -29,10 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,7 +210,7 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,7 +217,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -41,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -213,15 +220,8 @@ +@@ -220,15 +227,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -58,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -67,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env index 65467f110a..43cdfb1cfc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.env @@ -33,10 +33,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -205,6 +207,9 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -212,6 +214,9 @@ # a branch name or an environment name. VORTEX_FETCH_DB_ENVIRONMENT=prod @@ -46,7 +46,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -213,15 +218,8 @@ +@@ -220,15 +225,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -63,7 +63,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +236,7 @@ +@@ -245,7 +243,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -72,7 +72,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +263,3 @@ +@@ -272,17 +270,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml index a4f3384e34..bb8d3535a2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -271,6 +271,9 @@ +@@ -276,6 +276,9 @@ VORTEX_FETCH_DB_SEMAPHORE=/tmp/fetch-db-fresh ./vendor/bin/vortex-fetch-db echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env index 0689bf7b1b..ff319451f1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/hosting_project_name___lagoon/.env @@ -29,10 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,7 +210,7 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,7 +217,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -41,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -213,15 +220,8 @@ +@@ -220,15 +227,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -58,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -67,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env index a1d35bca50..cfd0cd8e9f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_disabled_lagoon/.env @@ -29,10 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,7 +210,7 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,7 +217,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -41,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -213,15 +220,8 @@ +@@ -220,15 +227,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -58,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -67,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env index ac0a01196b..9f07ea5d17 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,34 @@ +@@ -213,6 +213,41 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,9 +19,16 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. @@ -33,7 +40,7 @@ # FILES SOURCE # ################################################################################ -@@ -279,3 +307,6 @@ +@@ -286,3 +321,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml index 35ffec4ffe..1b850740a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -272,6 +272,9 @@ +@@ -277,6 +277,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +290,14 @@ +@@ -292,6 +295,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -23,7 +23,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +453,11 @@ +@@ -447,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml index 49de9d5ef1..559b4ce6a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -331,7 +338,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -607,6 +614,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -682,3 +691,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env index ac0a01196b..9f07ea5d17 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_circleci/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,34 @@ +@@ -213,6 +213,41 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,9 +19,16 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. @@ -33,7 +40,7 @@ # FILES SOURCE # ################################################################################ -@@ -279,3 +307,6 @@ +@@ -286,3 +321,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env index 005f07c4a4..cea0d09d1d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.env @@ -29,10 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,9 +210,32 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,9 +217,39 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -52,9 +52,16 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=lagoon + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. @@ -66,7 +73,7 @@ # FILES SOURCE # ################################################################################ -@@ -213,15 +243,8 @@ +@@ -220,15 +257,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -83,7 +90,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +261,7 @@ +@@ -245,7 +275,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -92,7 +99,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +288,3 @@ +@@ -272,17 +302,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml index 35ffec4ffe..1b850740a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_enabled_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -272,6 +272,9 @@ +@@ -277,6 +277,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +290,14 @@ +@@ -292,6 +295,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -23,7 +23,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +453,11 @@ +@@ -447,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env index 30942115aa..e94fca2354 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,29 @@ +@@ -213,6 +213,36 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -14,9 +14,16 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=acquia + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml index 35ffec4ffe..1b850740a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_acquia/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -272,6 +272,9 @@ +@@ -277,6 +277,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +290,14 @@ +@@ -292,6 +295,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -23,7 +23,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +453,11 @@ +@@ -447,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env index ae32230e6f..84d4b40fd6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,44 @@ +@@ -213,6 +213,51 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -29,9 +29,16 @@ +# in CI. +# VORTEX_REFRESH_DB2_SOURCE=url + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml index 38db031910..94f4ed490d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_container_registry/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -241,6 +241,10 @@ +@@ -246,6 +246,10 @@ { echo "VORTEX_CI_DB_CACHE_FALLBACK=no" echo "VORTEX_FRONTEND_BUILD_SKIP=1" @@ -6,10 +6,10 @@ + echo "VORTEX_DB_IMAGE_BASE=drevops/db-data-mysql-8.4:__VERSION__" + # Deploy container image. + echo "VORTEX_EXPORT_DB_CONTAINER_REGISTRY_PUSH_PROCEED=1" - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. echo "VORTEX_PUSH_DB_PROCEED=1" } >> "$GITHUB_ENV" -@@ -272,9 +276,13 @@ +@@ -277,9 +281,13 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -23,7 +23,7 @@ ./vendor/bin/vortex-refresh-db timeout-minutes: 30 -@@ -287,6 +295,14 @@ +@@ -292,6 +300,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -38,7 +38,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +458,11 @@ +@@ -447,6 +463,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env index aaa0ad57b2..3be11eb850 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,52 @@ +@@ -213,6 +213,52 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -30,16 +30,16 @@ +# Second database dump FTP file name. +VORTEX_FETCH_DB2_FTP_FILE=db2.sql + -+# Upstream source of the second database dump file, such as url or lagoon. ++# Database store of the second database dump file: ftp or s3. +# -+# Uncomment to make the second database fetch source a dump store: the -+# nightly CI job fetches the dump from this source and pushes it, as fetched, -+# to the fetch source, next to the primary dump. -+# VORTEX_REFRESH_DB2_SOURCE=url ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= + +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml index e4582dfc2a..de03baffec 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_ftp/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -191,6 +191,8 @@ +@@ -196,6 +196,8 @@ VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" @@ -7,7 +7,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -272,6 +274,9 @@ +@@ -277,6 +279,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -17,7 +17,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +292,14 @@ +@@ -292,6 +297,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -32,7 +32,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +455,11 @@ +@@ -447,6 +460,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env index ddd07958cb..f3c7fbb806 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,29 @@ +@@ -213,6 +213,36 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -14,9 +14,16 @@ +# Second database fetch source. +VORTEX_FETCH_DB2_SOURCE=lagoon + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml index 35ffec4ffe..1b850740a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_lagoon/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -272,6 +272,9 @@ +@@ -277,6 +277,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +290,14 @@ +@@ -292,6 +295,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -23,7 +23,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +453,11 @@ +@@ -447,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env index 129946b761..1f93586abd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,44 @@ +@@ -213,6 +213,44 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -22,16 +22,16 @@ +# AWS S3 region. +VORTEX_FETCH_DB2_S3_REGION=ap-southeast-2 + -+# Upstream source of the second database dump file, such as url or lagoon. ++# Database store of the second database dump file: ftp or s3. +# -+# Uncomment to make the second database fetch source a dump store: the -+# nightly CI job fetches the dump from this source and pushes it, as fetched, -+# to the fetch source, next to the primary dump. -+# VORTEX_REFRESH_DB2_SOURCE=url ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= + +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml index 436837f2c7..ff8b98eb8b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_s3/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -191,6 +191,8 @@ +@@ -196,6 +196,8 @@ VORTEX_CI_DB_CACHE_FALLBACK: "yes" # Which branch to use as a source of DB caches. VORTEX_CI_DB_CACHE_BRANCH: "develop" @@ -7,7 +7,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -272,6 +274,9 @@ +@@ -277,6 +279,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -17,7 +17,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +292,14 @@ +@@ -292,6 +297,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -32,7 +32,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +455,11 @@ +@@ -447,6 +460,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env index ac0a01196b..9f07ea5d17 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,34 @@ +@@ -213,6 +213,41 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,9 +19,16 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. @@ -33,7 +40,7 @@ # FILES SOURCE # ################################################################################ -@@ -279,3 +307,6 @@ +@@ -286,3 +321,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml index 35ffec4ffe..1b850740a1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_source_url/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -272,6 +272,9 @@ +@@ -277,6 +277,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -8,7 +8,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +290,14 @@ +@@ -292,6 +295,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -23,7 +23,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +453,11 @@ +@@ -447,6 +458,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.ahoy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.ahoy.yml new file mode 100644 index 0000000000..11f7a073e6 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.ahoy.yml @@ -0,0 +1,51 @@ +@@ -32,6 +32,7 @@ + cmd: | + export COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-${PWD##*/}} + export VORTEX_HOST_DB_PORT=$(docker compose port database 3306 2>/dev/null | cut -d : -f 2) ++ export VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SOLR_PORT=$(docker compose port solr 8983 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_SELENIUM_VNC_PORT=$(docker compose port chrome 7900 2>/dev/null | cut -d : -f 2) + export VORTEX_HOST_HAS_SEQUELACE=$(uname -a | grep -i -q darwin && mdfind -name 'Sequel Ace' 2>/dev/null | grep -q "Ace" && echo 1 || true) +@@ -45,6 +46,14 @@ + open "mysql://${DATABASE_USERNAME:-drupal}:${DATABASE_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB_PORT}/drupal" -a "Sequel Ace" \ + || echo "Not a supported OS or Sequel Ace is not installed." + ++ db2: ++ usage: Open DB2 in Sequel Ace. ++ cmd: | ++ uname -a | grep -i -q darwin && test -d "${VORTEX_HOST_SEQUELACE_PATH:-/Applications/Sequel Ace.app}" && \ ++ VORTEX_HOST_DB2_PORT=$(docker compose port database2 3306 2>/dev/null | cut -d : -f 2) && \ ++ open "mysql://${DATABASE2_USERNAME:-drupal}:${DATABASE2_PASSWORD:-drupal}@127.0.0.1:${VORTEX_HOST_DB2_PORT}/drupal" -a "Sequel Ace" \ ++ || echo "Not a supported OS or Sequel Ace is not installed." ++ + # ---------------------------------------------------------------------------- + # Container commands. + # ---------------------------------------------------------------------------- +@@ -122,6 +131,13 @@ + case " $* " in *" --fresh "*) export VORTEX_FETCH_DB_FRESH=1 VORTEX_FETCH_DB_FORCE=1;; esac + ./vendor/bin/vortex-fetch-db + ++ fetch-db2: ++ usage: Fetch second database (migration). Run with "--fresh" to force. ++ aliases: [download-db2, db2-download, db2-fetch] ++ cmd: | ++ case " $* " in *" --fresh "*) export VORTEX_FETCH_DB2_FRESH=1 VORTEX_FETCH_DB2_FORCE=1;; esac ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + fetch-files: + usage: Fetch public files. + aliases: [download-files, files-download, files-fetch] +@@ -134,6 +150,13 @@ + docker compose rm --force --stop --volumes database + docker compose build database + ahoy up ++ ++ reload-db2: ++ usage: Reload the migration database container using local database image. ++ cmd: | ++ ahoy confirm "Running this command will replace your current database. Are you sure?" || exit 0 ++ docker compose rm --force --stop --volumes database2 ++ ahoy up -- --build database2 + + provision: + usage: Provision a site from the database dump or profile. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.env b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.env new file mode 100644 index 0000000000..201ff25191 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.env @@ -0,0 +1,65 @@ +@@ -213,6 +213,57 @@ + VORTEX_FETCH_DB_ENVIRONMENT=prod + + ################################################################################ ++# SECOND DATABASE (MIGRATION) # ++################################################################################ ++ ++# Second database for migrations. ++# See settings.migration.php for database credentials. ++ ++# Second database dump file name. ++VORTEX_FETCH_DB2_FILE=db2.sql ++ ++# Second database fetch source. ++VORTEX_FETCH_DB2_SOURCE=url ++ ++# Second database dump file sourced from a URL. ++# ++# HTTP Basic Authentication credentials should be embedded into the value. ++VORTEX_FETCH_DB2_URL= ++ ++# Second database dump file sourced from FTP. ++ ++# Second database dump FTP host. ++VORTEX_FETCH_DB2_FTP_HOST= ++ ++# Second database dump FTP port. ++VORTEX_FETCH_DB2_FTP_PORT=21 ++ ++# Second database dump FTP directory. ++# ++# Leave empty for the root directory of the FTP account. ++VORTEX_FETCH_DB2_FTP_DIR= ++ ++# Second database dump FTP file name. ++VORTEX_FETCH_DB2_FTP_FILE=db2.sql ++ ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE=ftp ++ ++# Destinations the nightly CI job pushes the second database dump to, as ++# fetched, besides the database store: ftp, s3 or both. ++# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 ++ ++# Environment to fetch the second database from. ++# ++# Applies to hosting environments. ++VORTEX_FETCH_DB2_ENVIRONMENT=prod ++ ++################################################################################ + # FILES SOURCE # + ################################################################################ + +@@ -286,3 +337,6 @@ + + # URL of the database used for demonstration with URL database fetch type. + VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql ++ ++# URL of the migration source database used for demonstration. ++VORTEX_FETCH_DB2_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db_d11.demo_source.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.github/workflows/build-test-deploy.yml new file mode 100644 index 0000000000..de03baffec --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/.github/workflows/build-test-deploy.yml @@ -0,0 +1,46 @@ +@@ -196,6 +196,8 @@ + VORTEX_CI_DB_CACHE_FALLBACK: "yes" + # Which branch to use as a source of DB caches. + VORTEX_CI_DB_CACHE_BRANCH: "develop" ++ VORTEX_FETCH_DB2_FTP_USER: ${{ secrets.VORTEX_FETCH_DB2_FTP_USER }} ++ VORTEX_FETCH_DB2_FTP_PASS: ${{ secrets.VORTEX_FETCH_DB2_FTP_PASS }} + + steps: + # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. +@@ -277,6 +279,9 @@ + echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" + timeout-minutes: 30 + ++ - name: Fetch migration DB ++ run: VORTEX_DB_INDEX=2 ./vendor/bin/vortex-fetch-db ++ + - name: Export DB + run: | + if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi +@@ -292,6 +297,14 @@ + ./vendor/bin/vortex-push-db + timeout-minutes: 30 + ++ # Push the freshly-fetched migration database dump to its push destinations, ++ # next to the primary dump. No-op unless the push was requested. ++ - name: Push migration DB ++ run: | ++ if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> No freshly fetched database. DB push will not proceed."; exit 0; fi ++ VORTEX_DB_INDEX=2 ./vendor/bin/vortex-push-db ++ timeout-minutes: 30 ++ + # Save cache per default branch and the timestamp. + # The cache will not be saved if it already exists. + # Note that the cache fallback flag is enabled for this case in order +@@ -447,6 +460,11 @@ + docker compose exec cli mkdir -p .data + docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql + rm -f .data/db.sql ++ fi ++ if [ -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ]; then ++ docker compose exec -T cli mkdir -p .data ++ docker compose --progress quiet cp -L ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" cli:"/app/.data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" ++ rm -f ".data/${VORTEX_FETCH_DB2_FILE:-db2.sql}" + fi + docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision + timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/composer.json b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/composer.json new file mode 100644 index 0000000000..b90a23c71c --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/composer.json @@ -0,0 +1,9 @@ +@@ -26,6 +26,8 @@ + "drupal/linkit": "__VERSION__", + "drupal/menu_trail_by_path": "__VERSION__", + "drupal/metatag": "__VERSION__", ++ "drupal/migrate_plus": "__VERSION__", ++ "drupal/migrate_tools": "__VERSION__", + "drupal/navigation_extra_tools": "__VERSION__", + "drupal/pathauto": "__VERSION__", + "drupal/redirect": "__VERSION__", diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/docker-compose.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/docker-compose.yml new file mode 100644 index 0000000000..70ac2fcbc8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/docker-compose.yml @@ -0,0 +1,52 @@ +@@ -57,6 +57,16 @@ + DRUPAL_SHIELD_PASS: ${DRUPAL_SHIELD_PASS:-} + # Redis integration flag. + DRUPAL_REDIS_ENABLED: ${DRUPAL_REDIS_ENABLED:-} ++ # Local migration database host (not used in production). ++ DATABASE2_HOST: database2 ++ # Local migration database name (not used in production). ++ DATABASE2_NAME: drupal ++ # Local migration database user (not used in production). ++ DATABASE2_USERNAME: drupal ++ # Local migration database password (not used in production). ++ DATABASE2_PASSWORD: drupal ++ # Local migration database port (not used in production). ++ DATABASE2_PORT: 3306 + + # The default user under which the containers should run. + x-user: &default-user +@@ -155,6 +165,20 @@ + ports: + - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database 3306`. + ++ database2: ++ image: "${VORTEX_DB2_IMAGE:-uselagoon/mysql-8.4:__VERSION__}" # Use custom database image (if defined) or fallback to standard database image. ++ environment: ++ <<: *default-environment ++ MYSQL_DATABASE: drupal ++ MYSQL_USER: drupal ++ MYSQL_PASSWORD: drupal ++ <<: *default-user ++ # dclint disable-line no-duplicate-exported-ports # container-only port, published on a random host port ++ ports: ++ - '3306' # Database port in a container. Find port on host with `ahoy info` or `docker compose port database2 3306`. ++ labels: ++ lagoon.type: mariadb # See https://docs.lagoon.sh/using-lagoon-advanced/service-types/ ++ + redis: + image: uselagoon/valkey-8:__VERSION__ + +@@ -198,11 +222,13 @@ + depends_on: + - clamav + - database ++ - database2 + - redis + - solr + command: + - clamav:3310 + - database:3306 ++ - database2:3306 + - redis:6379 + - solr:8983 + - curl -s "http://solr:8983/solr/drupal/select?q=*:*&rows=0&wt=json" | grep response diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/scripts/provision-20-migration.sh b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/scripts/provision-20-migration.sh new file mode 100755 index 0000000000..3000594760 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/scripts/provision-20-migration.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +## +# Import migration database and run migrations. +# +# This script is called during site provisioning via provision.sh. +# Customize the migration names at the bottom of this file. +# +# shellcheck disable=SC2086 + +set -eu +[ "${VORTEX_DEBUG-}" = "1" ] && set -x + +# Skip all migrations. +DRUPAL_MIGRATION_SKIP="${DRUPAL_MIGRATION_SKIP:-0}" + +# Skip rollback of migrations before import. +DRUPAL_MIGRATION_ROLLBACK_SKIP="${DRUPAL_MIGRATION_ROLLBACK_SKIP:-1}" + +# Limit the number of entities to import. Set to 'all' to import all. +DRUPAL_MIGRATION_IMPORT_LIMIT="${DRUPAL_MIGRATION_IMPORT_LIMIT:-50}" + +# Update already imported entities during migration. +DRUPAL_MIGRATION_UPDATE="${DRUPAL_MIGRATION_UPDATE:-0}" + +# Feedback frequency for migration progress. +DRUPAL_MIGRATION_FEEDBACK="${DRUPAL_MIGRATION_FEEDBACK:-50}" + +# Import migration source database. Set to 1 to import, 0 to skip. +DRUPAL_MIGRATION_SOURCE_DB_IMPORT="${DRUPAL_MIGRATION_SOURCE_DB_IMPORT:-${VORTEX_PROVISION_OVERRIDE_DB:-0}}" + +# Table name to probe in the source database to verify it is not corrupted. +DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE="${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE:-categories}" + +# Disable the search server while migrations run. Set to 0 to keep indexing. +DRUPAL_MIGRATION_SEARCH_DISABLE="${DRUPAL_MIGRATION_SEARCH_DISABLE:-1}" + +# Directory with database dump file. +VORTEX_DB_DIR="${VORTEX_DB_DIR:-./.data}" + +# Migration database dump file name. +VORTEX_FETCH_DB2_FILE="${VORTEX_FETCH_DB2_FILE:-db2.sql}" + +# ------------------------------------------------------------------------------ + +# @formatter:off +info() { printf " ==> %s\n" "${1}"; } +note() { printf " %s\n" "${1}"; } +task() { printf " > %s\n" "${1}"; } +pass() { printf " + %s\n" "${1}"; } +fail() { printf " ! %s\n" "${1}"; exit "${2:-1}"; } +# @formatter:on + +drush() { ./vendor/bin/drush -y "$@"; } + +# ------------------------------------------------------------------------------ + +info "Started migration operations." + +environment="$(drush php:eval "print \Drupal\Core\Site\Settings::get('environment');")" +note "Environment: ${environment}" + +# Skip migrations in production. +if [ "${environment}" = "prod" ]; then + DRUPAL_MIGRATION_SKIP=1 +fi + +note "Migration skip: ${DRUPAL_MIGRATION_SKIP}" +note "Migration limit: ${DRUPAL_MIGRATION_IMPORT_LIMIT}" +note "Migration skip rollback: ${DRUPAL_MIGRATION_ROLLBACK_SKIP}" +note "Migration update: ${DRUPAL_MIGRATION_UPDATE}" +note "Migration feedback: ${DRUPAL_MIGRATION_FEEDBACK}" +note "Source DB import: ${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" +note "Search disable: ${DRUPAL_MIGRATION_SEARCH_DISABLE}" +echo + +if [ "${DRUPAL_MIGRATION_SKIP}" = "1" ]; then + info "Skipped migrations. DRUPAL_MIGRATION_SKIP is set to 1." + exit 0 +fi + +# Helper function to run a single migration with configured options. +run_migration() { + local migration_name="${1}" + shift + + drush migrate:reset-status "${migration_name}" || fail "Failed to reset migration status for ${migration_name}." + + task "Running migration: ${migration_name}." + local opts=() + + opts+=("--feedback=${DRUPAL_MIGRATION_FEEDBACK}") + + if printf '%s\n' "${DRUPAL_MIGRATION_IMPORT_LIMIT}" | grep -q '^[0-9][0-9]*$' && [ "${DRUPAL_MIGRATION_IMPORT_LIMIT}" -gt 0 ]; then + opts+=("--limit=${DRUPAL_MIGRATION_IMPORT_LIMIT}") + fi + + if [ "${DRUPAL_MIGRATION_UPDATE}" = "1" ]; then + opts+=("--update") + fi + + # Add any additional arguments passed to the function. + opts+=("$@") + + drush migrate:import "${opts[@]}" "${migration_name}" || { + drush migrate:messages "${migration_name}" + fail "Failed to run migration ${migration_name}." + } + + pass "Migrated: ${migration_name}." +} + +# Detect if existing migration source database is corrupted. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" != "1" ]; then + note "Source database import is set to be skipped. Checking existing database." + task "Probing for '${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}' table in the source database." + if drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + pass "Source database is intact." + else + note "Migration source database is corrupted or empty. Re-importing." + DRUPAL_MIGRATION_SOURCE_DB_IMPORT=1 + pass "Marked the source database for re-import." + fi +fi + +# Import the migration source database from the dump file. +if [ "${DRUPAL_MIGRATION_SOURCE_DB_IMPORT}" = "1" ]; then + task "Importing migration source database." + + [ ! -f "${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" ] && fail "Migration source database file not found. Please run 'ahoy fetch-db2'." + + drush sql:drop --database=migrate + # shellcheck disable=SC2091 + $(drush sql:connect --database=migrate) <"${VORTEX_DB_DIR}/${VORTEX_FETCH_DB2_FILE}" + + pass "Imported migration source database." +else + note "Using existing migration source database." +fi + +task "Verifying migration source database." +if ! drush sql:query --database=migrate "SELECT COUNT(*) FROM ${DRUPAL_MIGRATION_SOURCE_DB_PROBE_TABLE}" >/dev/null 2>&1; then + drush sql:query --database=migrate "SHOW TABLES;" + fail "Migration source database is corrupted." +fi +pass "Verified migration source database." + +# Enable custom migration modules. +task "Enabling migration modules." +drush pm:install ys_migrate +pass "Enabled migration modules." + +info "Starting migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Disabling Search API Solr server." + drush search-api:server-disable solr || true + pass "Disabled Search API Solr server." +fi + +if [ "${DRUPAL_MIGRATION_ROLLBACK_SKIP}" = "1" ]; then + note "Skipped rollback of all migrations." +else + task "Rolling back all migrations." + drush migrate:rollback --all || true + pass "Rolled back all migrations." +fi +echo + +# ------------------------------------------------------------------------------ +# Add your migrations below. +# ------------------------------------------------------------------------------ + +run_migration ys_migrate_categories + +echo +note "Finished migrations." + +if [ "${DRUPAL_MIGRATION_SEARCH_DISABLE}" = "1" ]; then + task "Enabling Search API Solr server." + drush search-api:server-enable solr || true + drush search-api:enable-all || true + pass "Enabled Search API Solr server." +fi + +drush migrate:status + +info "Finished migration operations." diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/DatabaseSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/DatabaseSettingsTest.php new file mode 100644 index 0000000000..956cd535d4 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/DatabaseSettingsTest.php @@ -0,0 +1,110 @@ +@@ -47,6 +47,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -74,6 +85,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -101,6 +123,17 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + +@@ -128,9 +161,55 @@ + 'prefix' => '', + ], + ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], + ], + ]; + ++ yield [ ++ [ ++ 'DATABASE2_NAME' => 'migrate_db_name', ++ 'DATABASE2_USERNAME' => 'migrate_db_user', ++ 'DATABASE2_PASSWORD' => 'migrate_db_pass', ++ 'DATABASE2_HOST' => 'migrate_db_host', ++ 'DATABASE2_PORT' => '3307', ++ ], ++ [ ++ 'default' => [ ++ 'default' => [ ++ 'database' => 'drupal', ++ 'username' => 'drupal', ++ 'password' => 'drupal', ++ 'host' => 'localhost', ++ 'port' => '3306', ++ 'charset' => 'utf8mb4', ++ 'collation' => 'utf8mb4_general_ci', ++ 'driver' => 'mysql', ++ 'prefix' => '', ++ ], ++ ], ++ 'migrate' => [ ++ 'default' => [ ++ 'database' => 'migrate_db_name', ++ 'username' => 'migrate_db_user', ++ 'password' => 'migrate_db_pass', ++ 'host' => 'migrate_db_host', ++ 'port' => '3307', ++ 'prefix' => '', ++ 'driver' => 'mysql', ++ ], ++ ], ++ ], ++ ]; + } + + } diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/EnvironmentSettingsTest.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/EnvironmentSettingsTest.php new file mode 100644 index 0000000000..8adb9246e8 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/EnvironmentSettingsTest.php @@ -0,0 +1,14 @@ +@@ -163,6 +163,13 @@ + $databases['default']['default']['collation'] = 'utf8_general_ci'; + $databases['default']['default']['driver'] = 'mysql'; + $databases['default']['default']['prefix'] = ''; ++ $databases['migrate']['default']['database'] = 'drupal'; ++ $databases['migrate']['default']['username'] = 'drupal'; ++ $databases['migrate']['default']['password'] = 'drupal'; ++ $databases['migrate']['default']['host'] = 'localhost'; ++ $databases['migrate']['default']['port'] = ''; ++ $databases['migrate']['default']['prefix'] = ''; ++ $databases['migrate']['default']['driver'] = 'mysql'; + $this->assertEquals($databases, $this->databases); + + // Verify key config overrides. diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/SettingsTestCase.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/SettingsTestCase.php new file mode 100644 index 0000000000..350cc85c04 --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/tests/phpunit/Drupal/SettingsTestCase.php @@ -0,0 +1,8 @@ +@@ -62,6 +62,7 @@ + const ALLOWED_ENV_VARS = [ + // Service variables. + 'DATABASE_', ++ 'DATABASE2_', + 'REDIS_', + 'CLAMAV_', + 'COMPOSE_', diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml new file mode 100644 index 0000000000..0494698e5d --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/migrations/ys_migrate_categories.yml @@ -0,0 +1,29 @@ +id: ys_migrate_categories +label: 'Migrate categories to tags' +migration_tags: + - ys_migrate + +source: + plugin: table + key: migrate + table_name: categories + id_fields: + id: + type: integer + fields: + id: id + name: name + description: description + +process: + name: name + description/value: description + description/format: + plugin: default_value + default_value: plain_text + vid: + plugin: default_value + default_value: tags + +destination: + plugin: 'entity:taxonomy_term' diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/ys_migrate.info.yml b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/ys_migrate.info.yml new file mode 100644 index 0000000000..96c67179ac --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/modules/custom/ys_migrate/ys_migrate.info.yml @@ -0,0 +1,9 @@ +name: star wars Migrate +type: module +description: Migrations for star wars site. +core_version_requirement: ^11 +package: star_wars +dependencies: + - drupal:migrate + - migrate_plus:migrate_plus + - migrate_tools:migrate_tools diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.migration.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.migration.php new file mode 100644 index 0000000000..660869b39f --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.migration.php @@ -0,0 +1,21 @@ + (getenv('DATABASE2_NAME') ?: getenv('DATABASE2_DATABASE')) ?: 'drupal', + 'username' => getenv('DATABASE2_USERNAME') ?: 'drupal', + 'password' => getenv('DATABASE2_PASSWORD') ?: 'drupal', + 'host' => getenv('DATABASE2_HOST') ?: 'localhost', + 'port' => getenv('DATABASE2_PORT') ?: '', + 'prefix' => '', + 'driver' => 'mysql', +]; diff --git a/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.php b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.php new file mode 100644 index 0000000000..c04331d75e --- /dev/null +++ b/.vortex/installer/tests/Fixtures/handler_process/migration_fetch_store_ftp/web/sites/default/settings.php @@ -0,0 +1,15 @@ +@@ -167,6 +167,14 @@ + } + + //////////////////////////////////////////////////////////////////////////////// ++/// MIGRATION SETTINGS /// ++//////////////////////////////////////////////////////////////////////////////// ++// @todo Migration. Remove when migration configuration is no longer required. ++if (file_exists($app_root . '/' . $site_path . '/settings.migration.php')) { ++ require $app_root . '/' . $site_path . '/settings.migration.php'; ++} ++ ++//////////////////////////////////////////////////////////////////////////////// + /// LOCAL OVERRIDE /// + //////////////////////////////////////////////////////////////////////////////// + // @see https://www.vortextemplate.com/docs/development/settings#local-overrides diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_dev_modules_storybook/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_devel_sdc_devel_generated_content_testmode_reroute_email/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_no_sdc_devel/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml index 210025dfb1..ec62616083 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/modules_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -323,7 +323,6 @@ +@@ -328,7 +328,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +509,6 @@ +@@ -515,18 +514,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/names/.env b/.vortex/installer/tests/Fixtures/handler_process/names/.env index 907f094118..2086b384a4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/names/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/names/.env @@ -42,7 +42,7 @@ # Put the site into a maintenance mode during site provisioning. VORTEX_PROVISION_USE_MAINTENANCE_MODE=1 -@@ -255,7 +255,7 @@ +@@ -262,7 +262,7 @@ # An email address to send notifications from. # # Applies to email notifications. @@ -51,7 +51,7 @@ # Email address(es) to send notifications to. # -@@ -264,7 +264,7 @@ +@@ -271,7 +271,7 @@ # Multiple names can be specified as a comma-separated list of email addresses # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env index 2cdcb2a1e6..78f05bd3bc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_all/.env @@ -1,4 +1,4 @@ -@@ -250,7 +250,7 @@ +@@ -257,7 +257,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy @@ -7,7 +7,7 @@ # An email address to send notifications from. # -@@ -265,6 +265,12 @@ +@@ -272,6 +272,12 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env index 1a30a0585a..c2b1507ee2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_github_only/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,7 @@ +@@ -257,21 +257,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env index 0491e177eb..fd7abef749 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_jira_only/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,10 @@ +@@ -257,21 +257,10 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env index 2b5368dd47..719aefd89e 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_newrelic_only/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,7 @@ +@@ -257,21 +257,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env index 1338f61c23..7fcbe79bbd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_none/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,7 @@ +@@ -257,21 +257,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env index bfda460b3e..1371ec144b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_slack_only/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,7 @@ +@@ -257,21 +257,7 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env index e2de45c9a0..9dda6bcbc3 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/notification_channels_webhook_only/.env @@ -1,4 +1,4 @@ -@@ -250,21 +250,10 @@ +@@ -257,21 +257,10 @@ # The channels of the notifications. # # A combination of comma-separated values: email,slack,newrelic,github,jira,webhook,diffy diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env b/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env index a1d35bca50..cfd0cd8e9f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_database_lagoon/.env @@ -29,10 +29,10 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source - # when it is a dump store. Each destination reads its own variables, such as -@@ -203,7 +210,7 @@ + # Database store of the sanitized database dump file: ftp or s3. + # + # When set, the database is fetched from the store instead of the source. The +@@ -210,7 +217,7 @@ # Applies to hosting environments. # Note that depending on the hosting provider, this variable may represent # a branch name or an environment name. @@ -41,7 +41,7 @@ ################################################################################ # FILES SOURCE # -@@ -213,15 +220,8 @@ +@@ -220,15 +227,8 @@ # # Hosting sources copy the files from VORTEX_FETCH_FILES_ENVIRONMENT, or from # VORTEX_FETCH_DB_ENVIRONMENT when it is not set. @@ -58,7 +58,7 @@ ################################################################################ # RELEASE VERSIONING # ################################################################################ -@@ -238,7 +238,7 @@ +@@ -245,7 +245,7 @@ # Deployment occurs when tests pass in the CI environment. # @see https://www.vortextemplate.com/docs/deployment @@ -67,7 +67,7 @@ ################################################################################ # NOTIFICATIONS # -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env index 3def98864d..ab3c09e5d5 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.env @@ -7,7 +7,7 @@ # Overwrite a database if it exists. # -@@ -179,32 +179,6 @@ +@@ -179,39 +179,6 @@ # The file is used to import the database into an empty database container. VORTEX_DB_FILE=db.sql @@ -19,11 +19,18 @@ -# HTTP Basic Authentication credentials should be embedded into the value. -VORTEX_FETCH_DB_URL= - --# Destinations the nightly CI job pushes the sanitized database dump to: --# ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source --# when it is a dump store. Each destination reads its own variables, such as --# VORTEX_PUSH_DB_FTP_HOST for ftp. The database is always fetched from 1 --# source. +-# Database store of the sanitized database dump file: ftp or s3. +-# +-# When set, the database is fetched from the store instead of the source. The +-# nightly CI job fetches from the source, sanitizes the database and pushes +-# the dump to the store. The store reads the FTP or S3 variables above. Leave +-# empty to fetch from the source. +-VORTEX_FETCH_DB_STORE= +- +-# Destinations the nightly CI job pushes the sanitized database dump to +-# besides the database store: ftp, s3 or both, such as "ftp,s3". Each +-# destination reads its own variables, such as VORTEX_PUSH_DB_FTP_HOST for ftp. +-# The database is always fetched from 1 place. -# VORTEX_PUSH_DB_DESTINATIONS=ftp,s3 - -# Also push a copy of each dump named with the time of the push, such as @@ -40,7 +47,7 @@ ################################################################################ # FILES SOURCE # ################################################################################ -@@ -265,17 +239,3 @@ +@@ -272,17 +239,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index f54fa2939e..a2b07f82cf 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -1,21 +1,26 @@ -@@ -52,8 +52,6 @@ +@@ -52,13 +52,6 @@ description: 'Enable terminal session for CI jobs' required: false default: false +- refresh_db: +- type: boolean +- description: 'Refresh the database only' +- required: false +- default: false - schedule: - - cron: '0 18 * * *' defaults: run: -@@ -72,7 +70,6 @@ +@@ -77,7 +70,6 @@ lint: runs-on: ubuntu-latest -- if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} +- if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} container: # https://hub.docker.com/r/drevops/ci-runner -@@ -167,141 +164,8 @@ +@@ -172,141 +164,8 @@ run: docker compose exec -T cli bash -c "npm --prefix=\${WEBROOT}/themes/custom/\${DRUPAL_THEME} run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -88,12 +93,12 @@ - run: ./scripts/vortex-tooling.sh - - - name: Adjust variables for a scheduled run -- if: github.event_name == 'schedule' +- if: github.event_name == 'schedule' || inputs.refresh_db - run: | - { - echo "VORTEX_CI_DB_CACHE_FALLBACK=no" - echo "VORTEX_FRONTEND_BUILD_SKIP=1" -- # Refresh the database dumps in the fetch sources that are dump stores. +- # Push the sanitized database dump to the database store and destinations. - echo "VORTEX_PUSH_DB_PROCEED=1" - } >> "$GITHUB_ENV" - @@ -153,11 +158,11 @@ build: runs-on: ubuntu-latest - needs: database -- if: ${{ !inputs.deploy_target && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} +- if: ${{ !inputs.deploy_target && !inputs.refresh_db && github.event_name != 'schedule' && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} permissions: contents: read -@@ -344,14 +208,6 @@ +@@ -349,14 +208,6 @@ VORTEX_SSH_DISABLE_STRICT_HOST_KEY_CHECKING: "1" VORTEX_SSH_REMOVE_ALL_KEYS: "1" VORTEX_DEBUG: ${{ vars.VORTEX_DEBUG }} @@ -172,7 +177,7 @@ steps: # Frees disk space by removing preinstalled toolchains unused by this project. Steps run in a container, so the host is reached via the Docker socket. -@@ -390,29 +246,6 @@ +@@ -395,29 +246,6 @@ - name: Install Vortex tooling run: ./scripts/vortex-tooling.sh @@ -202,11 +207,11 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -588,7 +421,6 @@ +@@ -593,7 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] -- if: ${{ !cancelled() && (inputs.deploy_target || (success() && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} +- if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} permissions: contents: read diff --git a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env index 26fc5387c9..f33591edfa 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_cms_profile/.env @@ -7,7 +7,7 @@ # Drupal configuration directory. # -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env index e10eafb327..68b383de82 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/starter_drupal_profile/.env @@ -1,4 +1,4 @@ -@@ -265,17 +265,3 @@ +@@ -272,17 +272,3 @@ # with optional names in the format "email|name". # Example: "to1@example.com|Jane Doe, to2@example.com|John Doe" VORTEX_NOTIFY_EMAIL_RECIPIENTS="webmaster@star-wars.com|Webmaster" diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml index fb4ec49edc..f41c2e8005 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_claro/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -162,11 +162,6 @@ +@@ -167,11 +167,6 @@ run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -323,7 +318,6 @@ +@@ -328,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +504,6 @@ +@@ -515,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env b/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env index 5a3382abdd..2f4e32abbd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_custom_non_vortex/.env @@ -5,5 +5,5 @@ -VORTEX_FETCH_DB_URL= +VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql - # Destinations the nightly CI job pushes the sanitized database dump to: - # ftp, s3 or both, such as "ftp,s3". Defaults to the database fetch source + # Database store of the sanitized database dump file: ftp or s3. + # diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml index fb4ec49edc..f41c2e8005 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_olivero/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -162,11 +162,6 @@ +@@ -167,11 +167,6 @@ run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -323,7 +318,6 @@ +@@ -328,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +504,6 @@ +@@ -515,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml index fb4ec49edc..f41c2e8005 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/theme_stark/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -162,11 +162,6 @@ +@@ -167,11 +167,6 @@ run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -323,7 +318,6 @@ +@@ -328,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +504,6 @@ +@@ -515,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/timezone_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint/.github/workflows/build-test-deploy.yml index 96c349475a..49bfce98dd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -138,18 +138,6 @@ +@@ -143,18 +143,6 @@ run: docker compose exec -T cli composer normalize --dry-run continue-on-error: ${{ vars.VORTEX_CI_COMPOSER_NORMALIZE_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml index d8f25d774d..7fcb92fcd9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_lint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -305,7 +312,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -576,6 +583,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -651,3 +660,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml index b71d452b3f..e6ba2763e0 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -154,10 +154,6 @@ +@@ -159,10 +159,6 @@ run: docker compose exec -T cli vendor/bin/twig-cs-fixer continue-on-error: ${{ vars.VORTEX_CI_TWIG_CS_FIXER_IGNORE_FAILURE == '1' }} @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -322,14 +318,7 @@ +@@ -327,14 +323,7 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -24,7 +24,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -451,66 +440,6 @@ +@@ -456,66 +445,6 @@ run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} @@ -91,7 +91,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -523,19 +452,6 @@ +@@ -528,19 +457,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -111,7 +111,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -552,16 +468,6 @@ +@@ -557,16 +473,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml index bc5ca2b53f..61393ab578 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_be_tests_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -525,6 +532,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -600,3 +609,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml index 8116ff4305..6551a2f609 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -132,7 +132,6 @@ +@@ -137,7 +137,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -6,7 +6,7 @@ - name: Validate Composer configuration is normalized run: docker compose exec -T cli composer normalize --dry-run -@@ -158,15 +157,6 @@ +@@ -163,15 +162,6 @@ run: docker compose exec -T cli vendor/bin/gherkinlint lint tests/behat/features continue-on-error: ${{ vars.VORTEX_CI_GHERKIN_LINT_IGNORE_FAILURE == '1' }} @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -321,7 +311,6 @@ +@@ -326,7 +316,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -30,7 +30,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -432,7 +421,6 @@ +@@ -437,7 +426,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -38,7 +38,7 @@ - name: Provision site run: | -@@ -445,11 +433,6 @@ +@@ -450,11 +438,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml index ff41066688..e341c699ae 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -296,7 +303,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -559,6 +566,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -634,3 +643,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml index d10899d1e4..cab28b5a4c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -132,7 +132,6 @@ +@@ -137,7 +137,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -6,7 +6,7 @@ - name: Validate Composer configuration is normalized run: docker compose exec -T cli composer normalize --dry-run -@@ -158,15 +157,6 @@ +@@ -163,15 +162,6 @@ run: docker compose exec -T cli vendor/bin/gherkinlint lint tests/behat/features continue-on-error: ${{ vars.VORTEX_CI_GHERKIN_LINT_IGNORE_FAILURE == '1' }} @@ -22,7 +22,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -321,9 +311,7 @@ +@@ -326,9 +316,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -32,7 +32,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -432,7 +420,6 @@ +@@ -437,7 +425,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -40,7 +40,7 @@ - name: Provision site run: | -@@ -446,11 +433,6 @@ +@@ -451,11 +438,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -52,7 +52,7 @@ - name: Test with PHPUnit if: ${{ env.VORTEX_CI_IS_PHPUNIT_RUNNER == 'true' }} run: docker compose exec -T cli vendor/bin/phpunit -@@ -510,18 +492,6 @@ +@@ -515,18 +497,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml index d5693d7f8b..71fa1a182b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_groups_no_fe_lint_no_theme_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -287,7 +294,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -537,6 +544,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -612,3 +621,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml index d12b0c686a..7a9295885c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -154,10 +154,6 @@ +@@ -159,10 +159,6 @@ run: docker compose exec -T cli vendor/bin/twig-cs-fixer continue-on-error: ${{ vars.VORTEX_CI_TWIG_CS_FIXER_IGNORE_FAILURE == '1' }} @@ -9,7 +9,7 @@ - name: Lint module code with NodeJS linters run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} -@@ -324,12 +320,6 @@ +@@ -329,12 +325,6 @@ VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -22,7 +22,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -522,19 +512,6 @@ +@@ -527,19 +517,6 @@ exit 1 fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml index 090353bbea..7cbd60119b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_behat_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -566,6 +573,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -641,3 +650,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint/.github/workflows/build-test-deploy.yml index ada126edb3..9c2f6408e7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -120,10 +120,6 @@ +@@ -125,10 +125,6 @@ done continue-on-error: ${{ vars.VORTEX_CI_HADOLINT_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml index 96c7a95e27..c4e2655362 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -308,7 +315,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -579,6 +586,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -654,3 +663,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env index ac0a01196b..9f07ea5d17 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.env @@ -1,4 +1,4 @@ -@@ -206,6 +206,34 @@ +@@ -213,6 +213,41 @@ VORTEX_FETCH_DB_ENVIRONMENT=prod ################################################################################ @@ -19,9 +19,16 @@ +# HTTP Basic Authentication credentials should be embedded into the value. +VORTEX_FETCH_DB2_URL= + ++# Database store of the second database dump file: ftp or s3. ++# ++# When set, the second database is fetched from the store instead of the ++# source. The nightly CI job fetches from the source and pushes the dump, as ++# fetched, to the store, next to the primary dump. Leave empty to fetch from ++# the source. ++VORTEX_FETCH_DB2_STORE= ++ +# Destinations the nightly CI job pushes the second database dump to, as -+# fetched: ftp, s3 or both. Defaults to the second database fetch source when -+# it is a dump store. ++# fetched, besides the database store: ftp, s3 or both. +# VORTEX_PUSH_DB2_DESTINATIONS=ftp,s3 + +# Environment to fetch the second database from. @@ -33,7 +40,7 @@ # FILES SOURCE # ################################################################################ -@@ -279,3 +307,6 @@ +@@ -286,3 +321,6 @@ # URL of the database used for demonstration with URL database fetch type. VORTEX_FETCH_DB_URL=https://github.com/drevops/vortex/releases/download/__VERSION__/db.demo.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml index 5c873888ef..dc5350e178 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_dclint_migration/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -120,10 +120,6 @@ +@@ -125,10 +125,6 @@ done continue-on-error: ${{ vars.VORTEX_CI_HADOLINT_IGNORE_FAILURE == '1' }} @@ -9,7 +9,7 @@ - name: Build stack run: docker compose up --no-deps --detach cli -@@ -272,6 +268,9 @@ +@@ -277,6 +273,9 @@ echo "db_hash=${{ hashFiles('.data') }}" >> "$GITHUB_ENV" timeout-minutes: 30 @@ -19,7 +19,7 @@ - name: Export DB run: | if [ ! -f /tmp/fetch-db-fresh ]; then echo "==> Database fetch semaphore file is missing. DB export will not proceed."; exit 0; fi -@@ -287,6 +286,14 @@ +@@ -292,6 +291,14 @@ ./vendor/bin/vortex-push-db timeout-minutes: 30 @@ -34,7 +34,7 @@ # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. # Note that the cache fallback flag is enabled for this case in order -@@ -442,6 +449,11 @@ +@@ -447,6 +454,11 @@ docker compose exec cli mkdir -p .data docker compose --progress quiet cp -L .data/db.sql cli:/app/.data/db.sql rm -f .data/db.sql diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters/.github/workflows/build-test-deploy.yml index ef406cba06..ec633d5dcf 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -113,17 +113,6 @@ +@@ -118,17 +118,6 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml index 0aa494e797..4947eb56a7 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_docker_linters_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -301,7 +308,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -572,6 +579,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -647,3 +656,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_stylelint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_stylelint/.github/workflows/build-test-deploy.yml index 9ff95c0bff..1365834034 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_stylelint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_stylelint/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -132,7 +132,6 @@ +@@ -137,7 +137,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -6,7 +6,7 @@ - name: Validate Composer configuration is normalized run: docker compose exec -T cli composer normalize --dry-run -@@ -157,15 +156,6 @@ +@@ -162,15 +161,6 @@ - name: Lint code with Gherkin Lint run: docker compose exec -T cli vendor/bin/gherkinlint lint tests/behat/features continue-on-error: ${{ vars.VORTEX_CI_GHERKIN_LINT_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml index fb4ec49edc..f41c2e8005 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_eslint_no_theme/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -162,11 +162,6 @@ +@@ -167,11 +167,6 @@ run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -323,7 +318,6 @@ +@@ -328,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +504,6 @@ +@@ -515,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint/.github/workflows/build-test-deploy.yml index 6d774889fb..731de580cc 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -113,13 +113,6 @@ +@@ -118,13 +118,6 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml index a0c33f49b3..88b3320a26 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_hadolint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -310,7 +317,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -581,6 +588,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -656,3 +665,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml index d2ac4714a0..89a432e1f2 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -321,7 +321,6 @@ +@@ -326,7 +326,6 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. -@@ -432,7 +431,6 @@ +@@ -437,7 +436,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -14,7 +14,7 @@ - name: Provision site run: | -@@ -445,11 +443,6 @@ +@@ -450,11 +448,6 @@ fi docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml index 6344469422..88b8e61d8c 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_jest_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -580,6 +587,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -655,3 +664,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs/.github/workflows/build-test-deploy.yml index b6ca4aef6b..37d21c49a4 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -138,10 +138,6 @@ +@@ -143,10 +143,6 @@ run: docker compose exec -T cli composer normalize --dry-run continue-on-error: ${{ vars.VORTEX_CI_COMPOSER_NORMALIZE_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml index 1d990b3080..8abf6f4c0b 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpcs_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -584,6 +591,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -659,3 +668,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan/.github/workflows/build-test-deploy.yml index 11d8ae51eb..516d8c7142 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -142,10 +142,6 @@ +@@ -147,10 +147,6 @@ run: docker compose exec -T cli vendor/bin/phpcs continue-on-error: ${{ vars.VORTEX_CI_PHPCS_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml index 06fff5ff4e..e57ff6a764 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpstan_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -584,6 +591,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -659,3 +668,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml index 0459fb8811..743b013e7f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -322,7 +322,6 @@ +@@ -327,7 +327,6 @@ VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -6,7 +6,7 @@ VORTEX_CI_IS_SDC_DEVEL_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true -@@ -450,66 +449,6 @@ +@@ -455,66 +454,6 @@ if: ${{ env.VORTEX_CI_IS_JEST_RUNNER == 'true' }} run: docker compose exec -T cli bash -c "npm test" continue-on-error: ${{ vars.VORTEX_CI_JEST_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml index 95607bec4c..ed767f7a7f 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_phpunit_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -547,6 +554,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -622,3 +631,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector/.github/workflows/build-test-deploy.yml index dc3c9f074e..7f562195e9 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -146,10 +146,6 @@ +@@ -151,10 +151,6 @@ run: docker compose exec -T cli vendor/bin/phpstan continue-on-error: ${{ vars.VORTEX_CI_PHPSTAN_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml index 61b6f2744f..2a395d99aa 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_rector_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -584,6 +591,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -659,3 +668,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml index a863608ae8..a80a7a1a90 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -317,7 +324,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -588,6 +595,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -663,3 +672,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml index fb4ec49edc..f41c2e8005 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_stylelint_no_theme/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -162,11 +162,6 @@ +@@ -167,11 +167,6 @@ run: docker compose exec -T cli bash -c "npm run lint" continue-on-error: ${{ vars.VORTEX_CI_NODEJS_LINT_IGNORE_FAILURE == '1' }} @@ -10,7 +10,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -323,7 +318,6 @@ +@@ -328,7 +323,6 @@ VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} VORTEX_CI_IS_JEST_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} VORTEX_CI_IS_PHPUNIT_RUNNER: ${{ matrix.instance == 0 || strategy.job-total == 1 }} @@ -18,7 +18,7 @@ # Runs on every instance, using the `p` profile. VORTEX_CI_IS_BEHAT_RUNNER: true # Subtracted from the instance index to derive the profile number. Raise -@@ -510,18 +504,6 @@ +@@ -515,18 +509,6 @@ fi env: VORTEX_CI_CODE_COVERAGE_THRESHOLD: ${{ vars.VORTEX_CI_CODE_COVERAGE_THRESHOLD || '90' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig/.github/workflows/build-test-deploy.yml index d0b0b38e1f..9c8e0f5bf6 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -150,10 +150,6 @@ +@@ -155,10 +155,6 @@ run: docker compose exec -T cli vendor/bin/rector --dry-run continue-on-error: ${{ vars.VORTEX_CI_RECTOR_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml index 4b18ee9a34..1bc13f02e1 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_no_twig_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -313,7 +320,7 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' - # Refresh the database dumps in the fetch sources that are dump stores. + # Push the sanitized database dump to the database store and destinations. VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -584,6 +591,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -659,3 +668,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly diff --git a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml index 864bb33983..be73afbdfd 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/tools_none/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -113,17 +113,6 @@ +@@ -118,17 +118,6 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry @@ -16,7 +16,7 @@ - name: Build stack run: docker compose up --no-deps --detach cli -@@ -132,41 +121,11 @@ +@@ -137,41 +126,11 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -58,7 +58,7 @@ database: runs-on: ubuntu-latest if: ${{ !inputs.deploy_target && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')) }} -@@ -321,15 +280,7 @@ +@@ -326,15 +285,7 @@ # https://www.vortextemplate.com/docs/continuous-integration#test-parallelism VORTEX_CI_RUNNER_INDEX: ${{ strategy.job-index }} VORTEX_CI_RUNNER_TOTAL: ${{ strategy.job-total }} @@ -74,7 +74,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -432,7 +383,6 @@ +@@ -437,7 +388,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli bash -c " \ if [ -n \"\${PACKAGE_TOKEN:-}\" ]; then composer config --global --auth github-oauth.github.com \"\${PACKAGE_TOKEN}\"; fi && \ COMPOSER_MEMORY_LIMIT=-1 composer --ansi install --prefer-dist" @@ -82,7 +82,7 @@ - name: Provision site run: | -@@ -446,71 +396,6 @@ +@@ -451,71 +401,6 @@ docker compose exec $(env | cut -f1 -d= | sed 's/^/-e /') -T cli ./vendor/bin/vortex-provision timeout-minutes: 30 @@ -154,7 +154,7 @@ - name: Validate Single Directory Components if: ${{ env.VORTEX_CI_IS_SDC_DEVEL_RUNNER == 'true' }} run: | -@@ -523,19 +408,6 @@ +@@ -528,19 +413,6 @@ fi continue-on-error: ${{ vars.VORTEX_CI_SDC_DEVEL_IGNORE_FAILURE == '1' }} @@ -174,7 +174,7 @@ - name: Process test logs and artifacts if: always() run: | -@@ -552,16 +424,6 @@ +@@ -557,16 +429,6 @@ path: .logs include-hidden-files: true if-no-files-found: error diff --git a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env index fd79e009be..9128f1f277 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_other/.env @@ -1,4 +1,4 @@ -@@ -230,7 +230,7 @@ +@@ -237,7 +237,7 @@ # # Can be one of: calver, semver, other # @see https://www.vortextemplate.com/docs/releasing diff --git a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env index e01d4eb267..a9c3058373 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env +++ b/.vortex/installer/tests/Fixtures/handler_process/version_scheme_semver/.env @@ -1,4 +1,4 @@ -@@ -230,7 +230,7 @@ +@@ -237,7 +237,7 @@ # # Can be one of: calver, semver, other # @see https://www.vortextemplate.com/docs/releasing From 20593787cbd112d8768984ed23fb8e5cd3b75c70 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 18:29:52 +1100 Subject: [PATCH 37/40] Updated snapshots. --- .../.github/workflows/build-test-deploy.yml | 4 +-- .../.circleci/config.yml | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled/.github/workflows/build-test-deploy.yml index aa349c0f82..c0c9dbac48 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -319,6 +319,7 @@ +@@ -335,6 +335,7 @@ # it by one for every leading instance that does not run Behat, so the # first Behat instance still selects the `p0` catch-all profile. VORTEX_CI_BEHAT_PROFILE_OFFSET: 0 @@ -6,7 +6,7 @@ container: # https://hub.docker.com/r/drevops/ci-runner -@@ -523,6 +524,17 @@ +@@ -539,6 +540,17 @@ env: VORTEX_CI_BEHAT_PROFILE: ${{ vars.VORTEX_CI_BEHAT_PROFILE }} continue-on-error: ${{ vars.VORTEX_CI_BEHAT_IGNORE_FAILURE == '1' }} diff --git a/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled_circleci/.circleci/config.yml b/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled_circleci/.circleci/config.yml index 57684e83a9..0a1b950f1d 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled_circleci/.circleci/config.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/lighthouse_enabled_circleci/.circleci/config.yml @@ -9,6 +9,13 @@ # The "runner" container uses Docker client to control the remote Docker server. version: '2.1' +parameters: + # Refresh the database as on the nightly run, without building or deploying. + # Set it when triggering a pipeline from the CircleCI UI or API. + refresh_db: + type: boolean + default: false + aliases: # SSH key fingerprint to fetch the database. # Replace this key fingerprint with your own and remove this comment. @@ -289,6 +296,16 @@ jobs: ./vendor/bin/vortex-refresh-db no_output_timeout: 30m + # Push the freshly-exported sanitized database dump to the push + # destinations. No-op unless a fresh DB was fetched (the fetch semaphore) and + # the push was requested (VORTEX_PUSH_DB_PROCEED, in the script). + - run: + name: Push DB + command: | + [ ! -f /tmp/fetch-db-fresh ] && echo "==> No freshly fetched database. DB push will not proceed." && exit 0 + ./vendor/bin/vortex-push-db + no_output_timeout: 30m + - save_cache: # Save cache per default branch and the timestamp. # The cache will not be saved if it already exists. @@ -307,6 +324,8 @@ jobs: VORTEX_DEPLOY_SSH_FINGERPRINT: *deploy_ssh_fingerprint # Enforce fresh DB build (do not rely on fallback caches). VORTEX_CI_DB_CACHE_FALLBACK: 'no' + # Push the sanitized database dump to the database store and destinations. + VORTEX_PUSH_DB_PROCEED: 1 # Do not build the Drupal front-end. VORTEX_FRONTEND_BUILD_SKIP: 1 @@ -586,6 +605,8 @@ jobs: workflows: # Commit workflow. Runs for every commit push to the remote repository. commit: + when: + not: << pipeline.parameters.refresh_db >> jobs: - database: filters: @@ -661,3 +682,10 @@ workflows: - develop jobs: - database-nightly + + # Database refresh workflow runs the nightly database job on demand, when a + # pipeline is triggered with the "refresh_db" parameter. + refresh-db: + when: << pipeline.parameters.refresh_db >> + jobs: + - database-nightly From 1cc0ae6bd0f755b6f80fd13ce2d65b53dd9e27c7 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 19:59:45 +1100 Subject: [PATCH 38/40] [#3340] Split the deploy job condition over 2 lines to stay within the workflow line length limit. --- .github/workflows/build-test-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index ae9cc5ef31..3c3c4cc155 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -777,7 +777,9 @@ jobs: runs-on: ubuntu-latest needs: [build, lint] #;< !PROVISION_TYPE_PROFILE - if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} + if: >- + ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' + && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} #;> !PROVISION_TYPE_PROFILE permissions: From 8c3836528113b66cda131a1fd677bca40e02b9e4 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 20:01:32 +1100 Subject: [PATCH 39/40] Updated snapshots. --- .../_baseline/.github/workflows/build-test-deploy.yml | 4 +++- .../.github/workflows/build-test-deploy.yml | 6 ++++-- .../.github/workflows/build-test-deploy.yml | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml index 6c5c9f847b..e3e1640f53 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/_baseline/.github/workflows/build-test-deploy.yml @@ -593,7 +593,9 @@ jobs: deploy: runs-on: ubuntu-latest needs: [build, lint] - if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} + if: >- + ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' + && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} permissions: contents: read diff --git a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml index fcb094be56..09fd95d37a 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/deploy_types_none_gha/.github/workflows/build-test-deploy.yml @@ -1,4 +1,4 @@ -@@ -589,104 +589,3 @@ +@@ -589,106 +589,3 @@ timeout-minutes: 120 # Cancel the action after 120 minutes, regardless of whether a connection has been established. with: detached: true @@ -6,7 +6,9 @@ - deploy: - runs-on: ubuntu-latest - needs: [build, lint] -- if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} +- if: >- +- ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' +- && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} - - permissions: - contents: read diff --git a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml index a2b07f82cf..d310a2e461 100644 --- a/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml +++ b/.vortex/installer/tests/Fixtures/handler_process/provision_profile/.github/workflows/build-test-deploy.yml @@ -207,11 +207,13 @@ - name: Login to container registry run: ./vendor/bin/vortex-login-container-registry -@@ -593,7 +421,6 @@ +@@ -593,9 +421,6 @@ deploy: runs-on: ubuntu-latest needs: [build, lint] -- if: ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} +- if: >- +- ${{ !cancelled() && (inputs.deploy_target || (success() && !inputs.refresh_db && github.event_name != 'schedule' +- && !startsWith(github.head_ref || github.ref_name, 'deps/') && (github.event_name == 'push' || !startsWith(github.head_ref, 'project/')))) }} permissions: contents: read From dbc3fa196f41837724006b37bd6aa31b08a3e6a6 Mon Sep 17 00:00:00 2001 From: Alex Skrypnyk Date: Mon, 16 Nov 2026 20:08:28 +1100 Subject: [PATCH 40/40] [#3340] Re-recorded the installer video with the database store prompt. --- .vortex/docs/static/img/installer.json | 447 ++++++++++++------------- .vortex/docs/static/img/installer.svg | 2 +- 2 files changed, 220 insertions(+), 229 deletions(-) diff --git a/.vortex/docs/static/img/installer.json b/.vortex/docs/static/img/installer.json index f2b0401b6d..d2d7d0a79b 100644 --- a/.vortex/docs/static/img/installer.json +++ b/.vortex/docs/static/img/installer.json @@ -1,228 +1,219 @@ -{"version":2,"width":80,"height":27,"timestamp":1794812735,"command":"/home/user/demo/installer.exp","title":"Vortex Installer Demo","env":{"SHELL":"/opt/homebrew/opt/bash/bin/bash"}} -[0.506083, "o", "\u001b[36m──────────────────────────────────────────────────────────────────────────────\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m ██╗ ██╗ ██████╗ ██████╗ ████████╗ ███████╗ ██╗ ██╗\u001b[39m\r\r\n\u001b[36m ██║ ██║ ██╔═══██╗ ██╔══██╗ ╚══██╔══╝ ██╔════╝ ╚██╗██╔╝\u001b[39m\r\r\n\u001b[36m ██║ ██║ ██║ ██║ ██████╔╝ ██║ █████╗ ╚███╔╝\u001b[39m\r\r\n\u001b[36m ╚██╗ ██╔╝ ██║ ██║ ██╔══██╗ ██║ ██╔══╝ ██╔██╗\u001b[39m\r\r\n\u001b[36m ╚████╔╝ ╚██████╔╝ ██║ ██║ ██║ "] -[0.506124, "o", " █"] -[0.506196, "o", "██████╗ ██╔╝ ██╗\u001b[39m\r\r\n\u001b[36m ╚═══╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m Drupal project template\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m by DrevOps\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m──────────────────────────────────────────────────────────────────────────────\u001b[39m\r\r\n\u001b[2m Installer version: development\u001b[22m\r\r\n"] -[0.507376, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mWelcome to the Vortex interactive installer\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m───────────────────────────────────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m This tool will guide you through installing the latest \u001b[4mdevelopment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m version of Vortex into your project. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m You will be asked a few questions to tailor the c"] -[0.507428, "o", "onfigu"] -[0.507467, "o", "ration to your \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m site. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m No changes will be made until you confirm everything at the end. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Press \u001b[33mCtrl+C\u001b[39m at any time to exit the installer.\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Press \u001b[33mCtrl+U\u001b[39m at any time to go back to the previous step.\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└───────────────────────────────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b"] -[0.507507, "o", "[0m\r\r\n"] -[0.50802, "o", "\u001b[36m◇\u001b[0m Validating repository and reference\r\r\n"] -[0.514269, "o", "\u001b[90m│\u001b[0m \u001b[2mChecking repository \"/home/user/vortex\" and reference \"HEAD\"\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Repository and reference validated successfully\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[0.529286, "o", "\u001b[?25l"] -[0.529322, "o", "\u001b[36m◇\u001b[0m Press any key to continue...\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[3.533775, "o", "\u001b[2A\r\u001b[J"] -[3.541866, "o", "\u001b[?25h"] -[3.55806, "o", "\u001b[?25l"] -[3.558114, "o", "\r\r\n\u001b[90m┌\u001b[0m \u001b[1mGeneral information\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[3.558448, "o", "\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[3.713548, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar war\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar w\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use"] -[3.714008, "o", " this name in the project an"] -[4.796506, "o", "d documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37msta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mst\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37ms\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[4.796806, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe "] -[4.796952, "o", "will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.800645, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r"] -[5.800915, "o", "\r\n\u001b[90m│\u001b[0m \u001b[2mWe will u"] -[5.801183, "o", "se this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.801318, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.802172, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.802356, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.802642, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.804507, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.804789, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805049, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805178, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805375, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805475, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805584, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805687, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805768, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.805895, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806015, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806087, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806193, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806289, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806418, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.80652, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806624, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806781, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806881, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.806978, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807044, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807143, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807254, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807345, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807458, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807563, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807662, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807768, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807877, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.807979, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808092, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mS\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808193, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSt\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.80829, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808386, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808501, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808579, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar W\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808698, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808805, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar War\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.808904, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.809002, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Site name (1/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mStar Wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[5.809351, "o", "\u001b[36m◇\u001b[0m Site machine name \u001b[2m(2/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name for the project directory and in the code.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[6.813008, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Site machine name (2/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[6.813325, "o", "\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mStar Wars Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[6.9667, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars Or\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars O\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar War\u001b[0"] -[6.966708, "o", "m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0"] -[8.117088, "o", "m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar W\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m O"] -[8.117309, "o", "rganization name \u001b[2m(3/33)"] -[8.117473, "o", "\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[9.121796, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSt\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mS\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r"] -[9.122235, "o", "\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0"] -[9.122465, "o", "m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[10.122928, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[10.12294, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[10.123054, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[11.128186, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[11.1282, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[11.12834, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[12.129143, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[12.129156, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[12.12929, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[13.133486, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[13.133494, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[13.133603, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[14.137628, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[14.137648, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[14.137798, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[15.139218, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] -[15.139547, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] -[15.139727, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[16.141838, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mR\u001b[0m\u001b[36"] -[16.141854, "o", "m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b["] -[16.141983, "o", "90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[17.146334, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRe\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mReb\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebe\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebel\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebell\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r"] -[17.146339, "o", "\r\n\u001b[90m│\u001b[0m \u001b[2mWe will u"] -[17.146434, "o", "se this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[18.150173, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebelli\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebellio\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebellion\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Organization name (3/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mRebellion\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Organization machine name \u001b[2m(4/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mrebellion\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the code.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Organization "] -[18.150181, "o", "machine name (4/33)\r\r\n\u001b[90m"] -[18.150305, "o", "│\u001b[0m \u001b[2mrebellion\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[19.154568, "o", "\u001b[36m◇\u001b[0m Public domain \u001b[2m(5/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar-wars.com\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mDomain name without protocol and trailing slash.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Public domain (5/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar-wars.com\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mDrupal\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m How would you like your site to be created on the first run? \u001b[2m(6/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mChoose how your site will be created the first time after this installer\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mfinishes:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDrupal, installed from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a new site by \u001b[4mpopulating a fresh database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m from one of the standard Drupal installation profiles.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDrupal CMS, installed from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a new"] -[19.154581, "o", " site by \u001b[4mpopulat"] -[19.154714, "o", "ing a fresh database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m from the Drupal CMS recipe.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDrupal, loaded from the demo database\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a site by \u001b[4mloading an existing demo database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m provided with the installer.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDrupal, installed from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDrupal CMS, installed from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Drupal, loaded from the demo database\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇. Applies only on the first run of the installer.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[20.154979, "o", "\u001b[22A\r\u001b[J\u001b[36m◆\u001b[0m How would you like your site to be created on the first run? (6/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mDrupal, loaded from the demo database\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Profile \u001b[2m(7/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Standard\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mMinimal\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDemo Umami\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCustom (next prompt)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select which Drupal profile to use.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Profile (7/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mStandard\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Modules \u001b[2m(8/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Coffee\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Config split\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Config update\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Devel\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Diff\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Drupal helpers\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Entity usage\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Environment indicator\r\r\n"] -[20.154989, "o", "\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m"] -[20.155088, "o", " Fast 404\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Field group\r\r\n\u001b[90m│\u001b[0m \u001b[2m↓ 19 more\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more modules.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[21.156323, "o", "\u001b[17A\r\u001b[J\u001b[36m◆\u001b[0m Modules (8/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCoffee, Config split, Config update, Devel, Diff, Drupal helpers, Entity\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2musage, Environment indicator, Fast 404, Field group, Generated content, Gin\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2madmin theme, Linkit, Menu trail by path, Metatag, Navigation extra tools,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mPathauto, Redirect, Reroute email, Robots.txt, Scheduled transitions,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mSchema.org Metatag, SDC Devel, Seckit, Shield, Stage file proxy, Testmode,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mViews bulk operations, XML Sitemap\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Custom modules prefix \u001b[2m(9/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2msw\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in custom modules\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Custom modules prefix (9/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2msw\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Custom modules \u001b[2m(10/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mSelect which custom modules to include in your project:\u001b[0m"] -[21.156722, "o", "\r\r\n\u001b[90m│\u001b[0m \u001b["] -[21.159694, "o", "2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mBase\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Starter module with deploy hooks and test scaffolding for\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Unit, Kernel, Functional, and FunctionalJavascript tests.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mSearch\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Custom Solr search integration module. Requires the Solr\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m service to be selected.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDemo\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Demonstrates how Vortex tooling works: includes a counter\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m block with CSS/JS, PHPUnit example tests across all test\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m types, and a Behat feature. Safe to remove on real projects.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Base - starter module with utilities and test scaffolding\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Search - custom Solr search integration\r\r\n\u001b[90m│\u001b[0m \u001b["] -[21.159712, "o", "32m◼\u001b[0m Demo "] -[22.162701, "o", "- counter block and example tests to demonstrate tooling\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more modules.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[22A\r\u001b[J\u001b[36m◆\u001b[0m Custom modules (10/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mBase - starter module with utilities and test scaffolding, Search - custom\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mSolr search integration, Demo - counter block and example tests to\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdemonstrate tooling\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Theme \u001b[2m(11/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOlivero\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mClaro\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mStark\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Custom (next prompt)\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select which Drupal theme to use.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Theme (11/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom (next prompt)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Custom theme machine name \u001b[2m(11.1)\u001b[0m\r\r\n\u001b[90m│\u001b"] -[22.162712, "o", "[0m \u001b[90m│\u001b[0m \r\r\n\u001b["] -[22.162847, "o", "90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mWe will use this name as a custom theme name\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n"] -[23.166936, "o", "\u001b[6A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Custom theme machine name (11.1)\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Build front-end assets in the container? \u001b[2m(11.2)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mDisable to build theme assets on the host or as part of deployment.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[6A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Build front-end assets in the container? (11.2)\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Storybook component library? \u001b[2m(11.3)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mComponent library for the custom theme, rendered by Drupal itself.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mStories are authored in Twig next to each component, so the library shows\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mthe same markup the site"] -[23.166953, "o", " renders.\u001b[0m\r\r\n\u001b"] -[23.167111, "o", "[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mA development server runs on demand with `ahoy storybook`. A static build\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mis produced during provisioning and served at `/storybook` in local, CI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mand development environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds Storybook packages to the theme and a step to provisioning.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] -[24.171704, "o", "\u001b[14A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Storybook component library? (11.3)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m├\u001b[0m \u001b[1mCode repository\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Repository provider \u001b[2m(12/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mVortex offers full automation with GitHub, while support for other providers\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mis limited.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m GitHub\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select your code repository provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Repository provider (12/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mGitHub\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Release versioning scheme \u001b[2m(13/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mChoose your versioning scheme:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mCalendar Versioning (CalVer)\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m \u001b[4myear.month.patch\u001b[0m\u001b[2;3m (E.g., \u001b[4m24.1.0\u001b[0m\u001b[2;3m)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m "] -[24.171716, "o", "https://calver.org\u001b[0m"] -[24.171831, "o", "\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mSemantic Versioning (SemVer)\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m \u001b[4mmajor.minor.patch\u001b[0m\u001b[2;3m (E.g., \u001b[4m1.0.0\u001b[0m\u001b[2;3m)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m https://semver.org\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mOther\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Custom versioning scheme of your choice.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Calendar Versioning (CalVer)\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mSemantic Versioning (SemVer)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select your version scheme.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[25.173004, "o", "\u001b[20A\r\u001b[J\u001b[36m◆\u001b[0m Release versioning scheme (13/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCalendar Versioning (CalVer)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mEnvironment\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Timezone \u001b[2m(14/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mEtc/UTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇, or start typing to select the timezone for your project.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Timezone (14/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Services \u001b[2m(15/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m ClamAV\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Solr\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Redis\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more services.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Services (15/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mClamAV, Redis, Solr\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Development tools \u001b[2m(16/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b"] -[25.173011, "o", "[36m❯\u001b[0m \u001b[32m◼\u001b[0m PHP C"] -[25.173175, "o", "odeSniffer\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m PHPStan\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Rector\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m ESLint\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Stylelint\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Jest\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m PHPUnit\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Behat\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Twig CS Fixer\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m DCLint\r\r\n\u001b[90m│\u001b[0m \u001b[2m↓ 1 more\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more tools.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[26.174669, "o", "\u001b[17A\r\u001b[J\u001b[36m◆\u001b[0m Development tools (16/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mBehat, DCLint, ESLint, Hadolint, Jest, PHP CodeSniffer, PHPStan, PHPUnit,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRector, Stylelint, Twig CS Fixer\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mHosting\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Hosting provider \u001b[2m(17/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mAcquia Cloud\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mLagoon\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m None\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select your hosting provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[26.174737, "o", "\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Hosting provider (17/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[27.179128, "o", "\u001b[36m◇\u001b[0m Custom web root directory \u001b[2m(18/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mweb\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom directory where the web server serves the site.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[28.184581, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Custom web root directory (18/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mweb\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mDeployment\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Deployment types \u001b[2m(19/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◻\u001b[0m Code artifact\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mLagoon webhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Custom webhook\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more deployment types.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[28.185065, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Deployment types (19/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom webhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[29.189559, "o", "\u001b[90m├\u001b[0m \u001b[1mWorkflow\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Provision type \u001b[2m(20/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mProvisioning sets up the site in an environment using an already assembled\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mcodebase.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mImport from database dump\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Provisions the site by importing a database dump\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m typically copied from production into lower\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mInstall from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Provisions the site by installing a fresh Drupal\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m site from a profile every time an environment is\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m created.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Import from database dump\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mInstall from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ "] -[29.189936, "o", "to select the provi"] -[29.190152, "o", "sion type.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[19A\r\u001b[J"] -[30.193407, "o", "\u001b[36m◆\u001b[0m Provision type (20/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mImport from database dump\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Database source \u001b[2m(20.1)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m URL download\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mFTP download\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mAcquia backup\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mLagoon environment\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mContainer registry\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mS3 bucket\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the database fetch source.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] -[31.201026, "o", "\u001b[12A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Database source (20.1)\r\r\n\u001b[90m│\u001b[0m \u001b[2mURL download\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[36m◇\u001b[0m Use a second database for migrations? \u001b[2m(21/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds a second database service for Drupal migrations.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[31.201329, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Use a second database for migrations? (21/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mNotifications\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Notification channels \u001b[2m(22/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Email\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mGitHub\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mJIRA\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mNew Relic\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mSlack\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mWebhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more notification channels.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[32.205985, "o", "\u001b[11A\r\u001b[J\u001b[36m◆\u001b[0m Notification channels (22/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mEmail\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mContinuous Integration\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[32.206326, "o", "\u001b[36m◇\u001b[0m Continuous Integration provider \u001b[2m(23/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m GitHub Actions\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCircleCI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the CI provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[33.208965, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Continuous Integration provider (23/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mGitHub Actions\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[33.212012, "o", "\u001b[36m◇\u001b[0m Visual regression testing with Diffy? \u001b[2m(24/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mDiffy-powered visual regression workflow to compare deployed environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mAutomatically triggers comparisons for dependency-update pull requests\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mmatching the configured branch pattern (`deps/*` by default).\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRuns comparisons for pull requests tagged with the `VR` label and posts a\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3msticky comment.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mAlso supports manual runs against any URL.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRequires a Diffy account.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[34.216477, "o", "\u001b[15A\r\u001b[J\u001b[36m◆\u001b[0m Visual regression testing with Diffy? (24/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[34.217141, "o", "\u001b[36m◇\u001b[0m Performance and SEO audits with Lighthouse? \u001b[2m(25/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mLighthouse audits of performance, accessibility, best practices and SEO for\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mthe pages you list or the pages in the sitemap.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRuns locally with `ahoy test-lighthouse` and in CI after the tests, writing\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mHTML and JSON reports for every page.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mScores can be compared with per-category budgets. The CI step is advisory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3muntil you make it blocking.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds a Lighthouse container, an Ahoy command and a CI step.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[35.222461, "o", "\u001b[14A\r\u001b[J\u001b[36m◆\u001b[0m Performance and SEO audits with Lighthouse? (25/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[35.222599, "o", "\u001b[36m◇\u001b[0m Scan for committed secrets with Gitleaks? \u001b[2m(26/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRuns in CI to detect hardcoded passwords, API keys and tokens.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[36.22477, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Scan for committed secrets with Gitleaks? (26/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[36.225144, "o", "\u001b[90m├\u001b[0m \u001b[1mAutomations\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[36.225969, "o", "\u001b[36m◇\u001b[0m Dependency updates provider \u001b[2m(27/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Renovate GitHub app\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mRenovate self-hosted in CI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the dependency updates provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[37.230331, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Dependency updates provider (27/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mRenovate GitHub app\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[37.230636, "o", "\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n"] -[37.234491, "o", "\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Auto-merge dependency updates \u001b[2m(27.1)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mPull requests in the selected groups are merged as soon as they meet the\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mbranch protection rules. Pull requests in other groups need a manual\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mmerge.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRequires \"Allow auto-merge\" in the repository settings and at least one\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mrequired status check on the base branch.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m PHP packages, except Drupal core (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Drupal core (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m JavaScript packages (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Container images (major, minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m CI tools (major, minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m GitHub Actions (major, minor and patch)\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2m"] -[37.234565, "o", "Use ⬆, ⬇ an"] -[37.234643, "o", "d Space bar to select one or more groups.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] -[38.238885, "o", "\u001b[17A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Auto-merge dependency updates (27.1)\r\r\n\u001b[90m│\u001b[0m \u001b[2mPHP packages, except Drupal core (minor and patch), Drupal core (minor and\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mpatch), JavaScript packages (minor and patch), Container images (major,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mminor and patch), CI tools (major, minor and patch), GitHub Actions\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m(major, minor and patch)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] -[38.239137, "o", "\u001b[36m◇\u001b[0m Code coverage provider \u001b[2m(28/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCodecov\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m None\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the code coverage provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[39.240952, "o", "\u001b[7A\r\u001b[J\u001b[36m◆\u001b[0m Code coverage provider (28/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[39.24145, "o", "\u001b[36m◇\u001b[0m Auto-assign the author to their PR? \u001b[2m(29/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to keep the PRs organized.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[40.245535, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Auto-assign the author to their PR? (29/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[40.245848, "o", "\u001b[36m◇\u001b[0m Auto-add a CONFLICT label to a PR when conflicts occur? \u001b[2m(30/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to keep quickly identify PRs that need attention.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[41.25086, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Auto-add a CONFLICT label to a PR when conflicts occur? (30/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[41.251074, "o", "\u001b[90m├\u001b[0m \u001b[1mDocumentation\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[41.251452, "o", "\u001b[36m◇\u001b[0m Preserve project documentation? \u001b[2m(31/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to maintain the project documentation within the repository.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[42.255717, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Preserve project documentation? (31/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mAI\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[42.255815, "o", "\u001b[36m◇\u001b[0m Provide AI agent instructions? \u001b[2m(32/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mProvides AI coding agents with better context about the project.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[43.25961, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Provide AI agent instructions? (32/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[43.260086, "o", "\u001b[36m◇\u001b[0m Configure AI code review with CodeRabbit? \u001b[2m(33/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRequires the CodeRabbit app on the repository.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[44.26217, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Configure AI code review with CodeRabbit? (33/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[44.272055, "o", "\u001b[?25h"] -[44.273225, "o", "\u001b[34m●\u001b[0m \u001b[34mInstallation summary\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[44.276508, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mGeneral information\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Site name \u001b[90m│\u001b[0m Star Wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Site machine name \u001b[90m│\u001b[0m star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Organization name \u001b[90m│\u001b[0m Rebellion \u001b[90m│\u001b[0m\r\r\n\u001b[90m"] -[44.276533, "o", "│\u001b[0"] -[44.276583, "o", "m \u001b[90m│\u001b[0m Organization machine name \u001b[90m│\u001b[0m rebellion \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Public domain \u001b[90m│\u001b[0m star-wars.com \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDrupal\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Starter \u001b[90m│\u001b[0m load_demodb \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b["] -[44.276624, "o", "90m│"] -[44.2767, "o", "\u001b[0m Modules \u001b[90m│\u001b[0m coffee, config_split, config_update, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m devel, diff, drupal_helpers, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m entity_usage, environment_indicator, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m fast_404, field_group, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m generated_content, gin, linkit, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m menu_trail_by_path, metatag, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m navigation_extra_tools, pathauto, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m redirect, reroute_email, robotstxt, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m "] -[44.276722, "o", " "] -[44.276796, "o", " \u001b[90m│\u001b[0m scheduled_transitions, schema_metatag, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m sdc_devel, seckit, shield, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m stage_file_proxy, testmode, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m views_bulk_operations, xmlsitemap \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Webroot \u001b[90m│\u001b[0m web \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Profile \u001b[90m│\u001b[0m standard \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Module prefix \u001b[90m│\u001b[0m sw \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Custom modules \u001b[90m│\u001b[0m base, search, demo \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Theme machine name "] -[44.276822, "o", " "] -[44.276903, "o", " \u001b[90m│\u001b[0m star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Build front-end in container \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Storybook component library \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mCode repository\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Code provider "] -[44.276925, "o", " \u001b[90m"] -[44.277028, "o", "│\u001b[0m github \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Version scheme \u001b[90m│\u001b[0m calver \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mEnvironment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Timezone \u001b[90m│\u001b[0m UTC \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Services \u001b[90m│\u001b["] -[44.277032, "o", "0m cla"] -[44.277121, "o", "mav, redis, solr \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Tools \u001b[90m│\u001b[0m behat, dclint, eslint, hadolint, jest, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m phpcs, phpstan, phpunit, rector, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m stylelint, twig_cs_fixer \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mHosting\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────"] -[44.277153, "o", "──"] -[44.277218, "o", "──────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Hosting provider \u001b[90m│\u001b[0m none \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDeployment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Deployment types \u001b[90m│\u001b[0m webhook \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├───────────────────────"] -[44.277221, "o", "──"] -[44.277301, "o", "─────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mWorkflow\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Provision type \u001b[90m│\u001b[0m database \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Database source \u001b[90m│\u001b[0m url \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Migration database \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────"] -[44.277305, "o", "───"] -[44.277396, "o", "─────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mNotifications\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Channels \u001b[90m│\u001b[0m email \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mContinuous Integration\u001b[0m \u001b["] -[44.2774, "o", "90m"] -[44.277492, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m CI provider \u001b[90m│\u001b[0m gha \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Visual regression testing \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Lighthouse audits \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Secret scanning with Gitleaks \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────"] -[44.277499, "o", "┤\u001b[0m\r\r\n\u001b"] -[44.277596, "o", "[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mAutomations\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Dependency updates provider \u001b[90m│\u001b[0m renovatebot_app \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Auto-merge dependency updates \u001b[90m│\u001b[0m php, drupal_core, js, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m container_images, ci_tools, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m github_actions \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Code coverage provider \u001b[90m│\u001b[0m none \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b"] -[44.277609, "o", "[0m \u001b["] -[44.277693, "o", "90m│\u001b[0m Auto-assign PR author \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Auto-add a CONFLICT label to PRs \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDocumentation\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Preserve project documentation \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m"] -[44.277698, "o", "├─"] -[44.277798, "o", "─────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mAI\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m AI agent instructions \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m AI code review \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴──────────"] -[44.277829, "o", "──"] -[44.277916, "o", "────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mLocations\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Current directory \u001b[90m│\u001b[0m /home/user/vortex/.artifa \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m cts/tmp/videos-workspace \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Destination directory \u001b[90m│\u001b[0m /home/user/vortex/.artifa \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m cts/tmp/videos-workspace/star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Vortex reposit"] -[44.277922, "o", "ory "] -[44.278014, "o", " \u001b[90m│\u001b[0m /home/user/vortex \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Vortex reference \u001b[90m│\u001b[0m HEAD \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└──────────────────────────────────┴────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[44.281142, "o", " Vortex will be installed into your project's directory \"/home/user/demo/star_wars\"\r\r\n"] -[44.294286, "o", "\u001b[?25l"] -[44.294341, "o", "\u001b[36m◇\u001b[0m Proceed with installing Vortex?\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[46.299583, "o", "\u001b[4A\r\u001b[J\u001b[36m◆\u001b[0m Proceed with installing Vortex?\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[46.309323, "o", "\u001b[?25h\u001b[34m●\u001b[0m \u001b[34mStarting project installation\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[46.30943, "o", "\u001b[36m◇\u001b[0m Downloading Vortex\r\r\n"] -[46.45556, "o", "\u001b[90m│\u001b[0m \u001b[2mDownloading from \"/home/user/vortex\" repository at ref \"HEAD\"\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Vortex downloaded (develop)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[46.455621, "o", "\u001b[36m◇\u001b[0m Customizing Vortex for your project\r\r\n"] -[47.417966, "o", "\u001b[90m│\u001b[0m \u001b[32m✓ Vortex was customized for your project\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[47.418033, "o", "\u001b[36m◇\u001b[0m Preparing destination directory\r\r\n"] -[47.431497, "o", "\u001b[90m│\u001b[0m \u001b[2mCreated directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mInitializing a new Git repository in directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Destination directory is ready\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[47.431596, "o", "\u001b[36m◇\u001b[0m Copying files to the destination directory\r\r\n"] -[47.599094, "o", "\u001b[90m│\u001b[0m \u001b[32m✓ Files copied to destination directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Preparing demo content\r\r\n"] -[48.126212, "o", "\u001b[90m│\u001b[0m \u001b[2mCreated data directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars/.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdata\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo database dump file was found in\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars/.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdata\" directory.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mFetched demo database from\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mhttps://github.com/drevops/vortex/releases/download/1.40.0/db.demo.sql.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Demo content prepared\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[48.126894, "o", "\u001b[90m│\u001b[0m \u001b[90m┌────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mFinished installing Vortex\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m──────────────────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Add and commit all files: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m git add -A \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m git commit -m \"Initial commit.\" \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Files: 402 added \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Status and diff of every file are recorded in: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m .logs/vortex-install-2026-11-16-070622"] -[48.126897, "o", ".md \u001b["] -[48.126987, "o", "90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[48.14198, "o", "\u001b[?25l"] -[48.14216, "o", "\u001b[36m◇\u001b[0m Run the site build now?\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mTakes ~5-10 min; output will be streamed. You can skip and run later with:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mahoy build\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[50.148649, "o", "\u001b[7A\r\u001b[J\u001b[36m◆\u001b[0m Run the site build now?\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[50.159428, "o", "\u001b[?25h"] -[50.159924, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mReady to build\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m──────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Build the site: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m ahoy build \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Setup GitHub Actions: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m"] -[50.159998, "o", " "] -[50.160048, "o", " \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m https://www.vortextemplate.com/docs/continuous-integration/github-actio \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m ns#onboarding \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└───────────────────────────────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] -[50.168864, "x", "0"] +{"version":2,"width":80,"height":27,"timestamp":1794820045,"command":"/home/user/demo/installer.exp","title":"Vortex Installer Demo","env":{"SHELL":"/opt/homebrew/opt/bash/bin/bash"}} +[0.565793, "o", "\u001b[36m──────────────────────────────────────────────────────────────────────────────\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m ██╗ ██╗ ██████╗ ██████╗ ████████╗ ███████╗ ██╗ ██╗\u001b[39m\r\r\n\u001b[36m ██║ ██║ ██╔═══██╗ ██╔══██╗ ╚══██╔══╝ ██╔════╝ ╚██╗██╔╝\u001b[39m\r\r\n\u001b[36m ██║ ██║ ██║ ██║ ██████╔╝ ██║ █████╗ ╚███╔╝\u001b[39m\r\r\n\u001b[36m ╚██╗ ██╔╝ ██║ ██║ ██╔══██╗ ██║ ██╔══╝ ██╔██╗\u001b[39m\r\r\n\u001b[36m ╚████╔╝ ╚██████╔╝ ██║ ██║ ██║ "] +[0.5658, "o", " █"] +[0.565941, "o", "██████╗ ██╔╝ ██╗\u001b[39m\r\r\n\u001b[36m ╚═══╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m Drupal project template\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m by DrevOps\u001b[39m\r\r\n\u001b[36m\u001b[39m\r\r\n\u001b[36m──────────────────────────────────────────────────────────────────────────────\u001b[39m\r\r\n\u001b[2m Installer version: development\u001b[22m\r\r\n"] +[0.567404, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mWelcome to the Vortex interactive installer\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m───────────────────────────────────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m This tool will guide you through installing the latest \u001b[4mdevelopment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m version of Vortex into your project. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m You will be asked a few questions to tailor the c"] +[0.567738, "o", "onfigu"] +[0.567858, "o", "ration to your \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m site. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m No changes will be made until you confirm everything at the end. \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Press \u001b[33mCtrl+C\u001b[39m at any time to exit the installer.\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Press \u001b[33mCtrl+U\u001b[39m at any time to go back to the previous step.\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└───────────────────────────────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b"] +[0.567991, "o", "[0m\r\r\n"] +[0.568367, "o", "\u001b[36m◇\u001b[0m Validating repository and reference\r\r\n"] +[0.575357, "o", "\u001b[90m│\u001b[0m \u001b[2mChecking repository \"/home/user/vortex\" and reference \"HEAD\"\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Repository and reference validated successfully\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[0.597999, "o", "\u001b[?25l\u001b[36m◇\u001b[0m Press any key to continue...\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[3.601042, "o", "\u001b[2A\r\u001b[J"] +[3.610818, "o", "\u001b[?25h"] +[3.627244, "o", "\u001b[?25l"] +[3.627352, "o", "\r\r\n\u001b[90m┌\u001b[0m \u001b[1mGeneral information\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[3.627548, "o", "\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[3.782807, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar war\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar w\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mstar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use"] +[3.78282, "o", " this name in the project an"] +[4.76826, "o", "d documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37msta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mst\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37ms\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[4.768275, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe "] +[4.768374, "o", "will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771346, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r"] +[5.771399, "o", "\r\n\u001b[90m│\u001b[0m \u001b[2mWe will u"] +[5.771501, "o", "se this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771554, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771647, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771747, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771821, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.771931, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772029, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772143, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772239, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772369, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772436, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772552, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772672, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772767, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.772869, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773023, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773143, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773277, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773413, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773507, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773605, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773705, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773851, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.773903, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774017, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774127, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774239, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.77436, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774488, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774604, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774719, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774831, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.774903, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775017, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.77514, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775236, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Site\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775338, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mS\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775447, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSt\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.77555, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775657, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775757, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775858, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar W\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.775955, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.776107, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar War\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.77622, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Site name \u001b[2m(1/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.776318, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Site name (1/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mStar Wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[5.776587, "o", "\u001b[36m◇\u001b[0m Site machine name \u001b[2m(2/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name for the project directory and in the code.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[6.780462, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Site machine name (2/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[6.780711, "o", "\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mStar Wars Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[6.935998, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars Or\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars O\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar War\u001b[0"] +[6.936043, "o", "m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0"] +[7.466138, "o", "m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar Wa\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar W\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar \u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mStar\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m O"] +[7.467184, "o", "rganization name \u001b[2m(3/33)"] +[7.468398, "o", "\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSta\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mSt\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mS\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name i"] +[7.468928, "o", "n the project and documentat"] +[8.474134, "o", "ion.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b["] +[8.474141, "o", "90m│\u001b[0m \u001b[90mE.g. My Org"] +[8.474248, "o", "\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[9.478217, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] +[9.478287, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[9.478385, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[10.481331, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] +[10.481345, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[10.481497, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[11.485089, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] +[11.485106, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[11.485224, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[12.487456, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] +[12.487467, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[12.487624, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[13.490776, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My O"] +[13.491294, "o", "rg\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m"] +[13.491441, "o", "│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b["] +[13.491526, "o", "0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m O"] +[14.494091, "o", "rganization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90mE.g. My Org\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mR\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRe\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mReb\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebe\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe "] +[14.494103, "o", "will use this name in the pr"] +[14.494222, "o", "oject and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[15.498202, "o", "\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebel\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebell\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebelli\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebellio\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◇\u001b[0m Organization name \u001b[2m(3/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[37mRebellion\u001b[0m\u001b[36m█\u001b[0m\r\r"] +[15.498209, "o", "\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m"] +[15.498321, "o", " \u001b[2mWe will use this name in the project and documentation.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[16.503293, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Organization name (3/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mRebellion\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Organization machine name \u001b[2m(4/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mrebellion\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in the code.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Organization machine name (4/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mrebellion\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Public domain \u001b[2m(5/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar-wars.com\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mDomain name without protocol and trailing slash.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Public domain (5/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mstar-wars.com\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mDrupal\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m How would you like your site to be created on the first run? \u001b[2m(6/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mChoose how your site will be created the first time after this installer\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mfinishes:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b["] +[16.503311, "o", "90m│\u001b[0m \u001b[2;3m ○ \u001b"] +[16.503406, "o", "[1mDrupal, installed from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a new site by \u001b[4mpopulating a fresh database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m from one of the standard Drupal installation profiles.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDrupal CMS, installed from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a new site by \u001b[4mpopulating a fresh database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m from the Drupal CMS recipe.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDrupal, loaded from the demo database\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Creates a site by \u001b[4mloading an existing demo database\u001b[0m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m provided with the installer.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDrupal, installed from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDrupal CMS, installed from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Drupal, loaded from the demo database\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90"] +[16.50342, "o", "m│\u001b[0m \u001b[2mUs"] +[17.508701, "o", "e ⬆ and ⬇. Applies only on the first run of the installer.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[22A\r\u001b[J\u001b[36m◆\u001b[0m How would you like your site to be created on the first run? (6/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mDrupal, loaded from the demo database\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Profile \u001b[2m(7/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Standard\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mMinimal\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mDemo Umami\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCustom (next prompt)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select which Drupal profile to use.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Profile (7/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mStandard\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Modules \u001b[2m(8/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Coffee\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Config split\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Config update\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Devel\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Diff\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Drupal helpers\r\r\n\u001b[90m│\u001b"] +[17.509413, "o", "[0m \u001b[32m◼\u001b[0m Entity u"] +[17.5096, "o", "sage\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Environment indicator\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Fast 404\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Field group\r\r\n\u001b[90m│\u001b[0m \u001b[2m↓ 19 more\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more modules.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[18.511695, "o", "\u001b[17A\r\u001b[J\u001b[36m◆\u001b[0m Modules (8/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCoffee, Config split, Config update, Devel, Diff, Drupal helpers, Entity\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2musage, Environment indicator, Fast 404, Field group, Generated content, Gin\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2madmin theme, Linkit, Menu trail by path, Metatag, Navigation extra tools,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mPathauto, Redirect, Reroute email, Robots.txt, Scheduled transitions,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mSchema.org Metatag, SDC Devel, Seckit, Shield, Stage file proxy, Testmode,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mViews bulk operations, XML Sitemap\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Custom modules prefix \u001b[2m(9/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2msw\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mWe will use this name in custom modules\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Custom modules prefix (9/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2msw\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Custom modules \u001b[2m(10/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mSelect which custom modules to include in your project:\u001b[0m"] +[18.512051, "o", "\r\r\n\u001b[90m│\u001b[0m \u001b["] +[18.512543, "o", "2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mBase\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Starter module with deploy hooks and test scaffolding for\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Unit, Kernel, Functional, and FunctionalJavascript tests.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mSearch\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Custom Solr search integration module. Requires the Solr\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m service to be selected.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mDemo\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Demonstrates how Vortex tooling works: includes a counter\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m block with CSS/JS, PHPUnit example tests across all test\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m types, and a Behat feature. Safe to remove on real projects.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Base - starter module with utilities and test scaffolding\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Search - custom Solr search integration\r\r\n\u001b[90m│\u001b[0m \u001b["] +[18.512837, "o", "32m◼\u001b[0m Demo "] +[19.516281, "o", "- counter block and example tests to demonstrate tooling\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more modules.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[22A\r\u001b[J\u001b[36m◆\u001b[0m Custom modules (10/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mBase - starter module with utilities and test scaffolding, Search - custom\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mSolr search integration, Demo - counter block and example tests to\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdemonstrate tooling\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Theme \u001b[2m(11/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOlivero\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mClaro\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mStark\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Custom (next prompt)\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select which Drupal theme to use.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Theme (11/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom (next prompt)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Custom theme machine name \u001b[2m(11.1)\u001b[0m\r\r\n\u001b[90m│\u001b"] +[19.516674, "o", "[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mWe will use this name as a custom theme name\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n"] +[20.521937, "o", "\u001b[6A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Custom theme machine name (11.1)\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mstar_wars\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Build front-end assets in the container? \u001b[2m(11.2)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mDisable to build theme assets on the host or as part of deployment.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[6A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Build front-end assets in the container? (11.2)\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Storybook component library? \u001b[2m(11.3)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mComponent library for the custom theme, rendered by Drupal itself.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mStories are authored in Twig next to each component, so the library shows\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mthe same markup the site"] +[20.522346, "o", " renders.\u001b[0m\r\r\n\u001b"] +[20.522488, "o", "[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mA development server runs on demand with `ahoy storybook`. A static build\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mis produced during provisioning and served at `/storybook` in local, CI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mand development environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds Storybook packages to the theme and a step to provisioning.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] +[21.528138, "o", "\u001b[14A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Storybook component library? (11.3)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m├\u001b[0m \u001b[1mCode repository\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Repository provider \u001b[2m(12/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mVortex offers full automation with GitHub, while support for other providers\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mis limited.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m GitHub\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select your code repository provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Repository provider (12/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mGitHub\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Release versioning scheme \u001b[2m(13/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mChoose your versioning scheme:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mCalendar Versioning (CalVer)\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m \u001b[4myear.month.patch\u001b[0m\u001b[2;3m (E.g., \u001b[4m24.1.0\u001b[0m\u001b[2;3m)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m "] +[21.528409, "o", "https://calver.org\u001b[0m"] +[21.528697, "o", "\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mSemantic Versioning (SemVer)\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m \u001b[4mmajor.minor.patch\u001b[0m\u001b[2;3m (E.g., \u001b[4m1.0.0\u001b[0m\u001b[2;3m)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m https://semver.org\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mOther\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Custom versioning scheme of your choice.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Calendar Versioning (CalVer)\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mSemantic Versioning (SemVer)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select your version scheme.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[22.53309, "o", "\u001b[20A\r\u001b[J\u001b[36m◆\u001b[0m Release versioning scheme (13/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCalendar Versioning (CalVer)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mEnvironment\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Timezone \u001b[2m(14/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mEtc/UTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇, or start typing to select the timezone for your project.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[22.533291, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Timezone (14/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mUTC\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[23.536975, "o", "\u001b[36m◇\u001b[0m Services \u001b[2m(15/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m ClamAV\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Solr\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Redis\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more services.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[24.541338, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Services (15/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mClamAV, Redis, Solr\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Development tools \u001b[2m(16/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m PHP CodeSniffer\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m PHPStan\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Rector\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m ESLint\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Stylelint\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Jest\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m PHPUnit\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Behat\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Twig CS Fixer\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m DCLint\r\r\n\u001b[90m│\u001b[0m \u001b[2m↓ 1 more\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more tools.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[25.545452, "o", "\u001b[17A\r\u001b[J\u001b[36m◆\u001b[0m Development tools (16/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mBehat, DCLint, ESLint, Hadolint, Jest, PHP CodeSniffer, PHPStan, PHPUnit,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRector, Stylelint, Twig CS Fixer\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mHosting\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Hosting provider \u001b[2m(17/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mAcquia Cloud\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mLagoon\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mOther\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m None\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select your hosting provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[26.549402, "o", "\u001b[9A\r\u001b[J\u001b[36m◆\u001b[0m Hosting provider (17/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Custom web root directory \u001b[2m(18/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mweb\u001b[0m\u001b[36m█\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom directory where the web server serves the site.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[27.553381, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Custom web root directory (18/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mweb\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mDeployment\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Deployment types \u001b[2m(19/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◻\u001b[0m Code artifact\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mLagoon webhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Custom webhook\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more deployment types.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[27.553476, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Deployment types (19/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mCustom webhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[28.55741, "o", "\u001b[90m├\u001b[0m \u001b[1mWorkflow\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Provision type \u001b[2m(20/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mProvisioning sets up the site in an environment using an already assembled\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mcodebase.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mImport from database dump\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Provisions the site by importing a database dump\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m typically copied from production into lower\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m ○ \u001b[1mInstall from profile\u001b[22m\u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m Provisions the site by installing a fresh Drupal\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m site from a profile every time an environment is\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m created.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Import from database dump\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mInstall from profile\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ "] +[28.557416, "o", "to select the provi"] +[28.55752, "o", "sion type.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[29.561936, "o", "\u001b[19A\r\u001b[J\u001b[36m◆\u001b[0m Provision type (20/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mImport from database dump\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Database source \u001b[2m(20.1)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m URL download\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mFTP download\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mAcquia backup\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mLagoon environment\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mContainer registry\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mS3 bucket\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the database fetch source.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] +[30.566564, "o", "\u001b[12A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Database source (20.1)\r\r\n\u001b[90m│\u001b[0m \u001b[2mURL download\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Database store \u001b[2m(20.1.1)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m None, fetch from the source\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mFTP server\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mS3 bucket\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select where developers fetch the sanitized database\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mfrom.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] +[31.572208, "o", "\u001b[9A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Database store (20.1.1)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNone, fetch from the source\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[36m◇\u001b[0m Use a second database for migrations? \u001b[2m(21/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds a second database service for Drupal migrations.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[31.572319, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Use a second database for migrations? (21/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mNotifications\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[32.575378, "o", "\u001b[36m◇\u001b[0m Notification channels \u001b[2m(22/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m Email\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mGitHub\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mJIRA\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mNew Relic\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mSlack\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m◻\u001b[0m \u001b[2mWebhook\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more notification channels.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[33.578448, "o", "\u001b[11A\r\u001b[J\u001b[36m◆\u001b[0m Notification channels (22/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mEmail\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mContinuous Integration\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Continuous Integration provider \u001b[2m(23/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m GitHub Actions\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCircleCI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the CI provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[33.578547, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Continuous Integration provider (23/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mGitHub Actions\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[34.582817, "o", "\u001b[36m◇\u001b[0m Visual regression testing with Diffy? \u001b[2m(24/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mDiffy-powered visual regression workflow to compare deployed environments.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mAutomatically triggers comparisons for dependency-update pull requests\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mmatching the configured branch pattern (`deps/*` by default).\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRuns comparisons for pull requests tagged with the `VR` label and posts a\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3msticky comment.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mAlso supports manual runs against any URL.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRequires a Diffy account.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[35.587013, "o", "\u001b[15A\r\u001b[J\u001b[36m◆\u001b[0m Visual regression testing with Diffy? (24/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Performance and SEO audits with Lighthouse? \u001b[2m(25/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mLighthouse audits of performance, accessibility, best practices and SEO for\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mthe pages you list or the pages in the sitemap.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRuns locally with `ahoy test-lighthouse` and in CI after the tests, writing\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mHTML and JSON reports for every page.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mScores can be compared with per-category budgets. The CI step is advisory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3muntil you make it blocking.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mAdds a Lighthouse container, an Ahoy command and a CI step.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[35.587116, "o", "\u001b[14A\r\u001b[J\u001b[36m◆\u001b[0m Performance and SEO audits with Lighthouse? (25/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[36.590983, "o", "\u001b[36m◇\u001b[0m Scan for committed secrets with Gitleaks? \u001b[2m(26/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRuns in CI to detect hardcoded passwords, API keys and tokens.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[37.595635, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Scan for committed secrets with Gitleaks? (26/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m├\u001b[0m \u001b[1mAutomations\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Dependency updates provider \u001b[2m(27/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m Renovate GitHub app\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mRenovate self-hosted in CI\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the dependency updates provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[38.601114, "o", "\u001b[8A\r\u001b[J\u001b[36m◆\u001b[0m Dependency updates provider (27/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mRenovate GitHub app\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[36m◇\u001b[0m Auto-merge dependency updates \u001b[2m(27.1)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mPull requests in the selected groups are merged as soon as they meet the\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mbranch protection rules. Pull requests in other groups need a manual\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mmerge.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3m\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mRequires \"Allow auto-merge\" in the repository settings and at least one\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2;3mrequired status check on the base branch.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[36m❯\u001b[0m \u001b[32m◼\u001b[0m PHP packages, except Drupal core (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Drupal core (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m JavaScript packages (minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m Container images (major, minor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m CI "] +[38.601742, "o", "tools (major, mi"] +[38.602869, "o", "nor and patch)\r\r\n\u001b[90m│\u001b[0m \u001b[32m◼\u001b[0m GitHub Actions (major, minor and patch)\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆, ⬇ and Space bar to select one or more groups.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n"] +[39.606724, "o", "\u001b[17A\r\u001b[J\u001b[90m│\u001b[0m \u001b[36m◆\u001b[0m Auto-merge dependency updates (27.1)\r\r\n\u001b[90m│\u001b[0m \u001b[2mPHP packages, except Drupal core (minor and patch), Drupal core (minor and\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mpatch), JavaScript packages (minor and patch), Container images (major,\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mminor and patch), CI tools (major, minor and patch), GitHub Actions\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m(major, minor and patch)\u001b[0m\r\r\n\u001b[90m│\u001b[0m \r\r\n\u001b[36m◇\u001b[0m Code coverage provider \u001b[2m(28/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mCodecov\u001b[0m\r\r\n\u001b[90m│\u001b[0m\u001b[36m❯\u001b[0m \u001b[32m●\u001b[0m None\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mUse ⬆ and ⬇ to select the code coverage provider.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[39.607693, "o", "\u001b[7A\r\u001b[J\u001b[36m◆\u001b[0m Code coverage provider (28/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNone\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Auto-assign the author to their PR? \u001b[2m(29/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to keep the PRs organized.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[40.614951, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Auto-assign the author to their PR? (29/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[36m◇\u001b[0m Auto-add a CONFLICT label to a PR when conflicts occur? \u001b[2m(30/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to keep quickly identify PRs that need attention.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[41.620282, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Auto-add a CONFLICT label to a PR when conflicts occur? (30/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[41.620319, "o", "\u001b[90m├\u001b[0m \u001b[1mDocumentation\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[41.620395, "o", "\u001b[36m◇\u001b[0m Preserve project documentation? \u001b[2m(31/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mHelps to maintain the project documentation within the repository.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[42.624663, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Preserve project documentation? (31/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[42.624729, "o", "\u001b[90m├\u001b[0m \u001b[1mAI\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[42.624841, "o", "\u001b[36m◇\u001b[0m Provide AI agent instructions? \u001b[2m(32/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mProvides AI coding agents with better context about the project.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[43.630387, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Provide AI agent instructions? (32/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[43.630511, "o", "\u001b[36m◇\u001b[0m Configure AI code review with CodeRabbit? \u001b[2m(33/33)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mRequires the CodeRabbit app on the repository.\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[44.633811, "o", "\u001b[6A\r\u001b[J\u001b[36m◆\u001b[0m Configure AI code review with CodeRabbit? (33/33)\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[44.64247, "o", "\u001b[?25h"] +[44.643738, "o", "\u001b[34m●\u001b[0m \u001b[34mInstallation summary\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[44.647212, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mGeneral information\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Site name \u001b[90m│\u001b[0m Star Wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Site machine name \u001b[90m│\u001b[0m star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Organization name \u001b[90m│\u001b[0m Rebellion \u001b[90m│\u001b[0m\r\r\n\u001b[90m"] +[44.647268, "o", "│\u001b[0"] +[44.647321, "o", "m \u001b[90m│\u001b[0m Organization machine name \u001b[90m│\u001b[0m rebellion \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Public domain \u001b[90m│\u001b[0m star-wars.com \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDrupal\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Starter \u001b[90m│\u001b[0m load_demodb \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b["] +[44.647364, "o", "90m│"] +[44.647432, "o", "\u001b[0m Modules \u001b[90m│\u001b[0m coffee, config_split, config_update, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m devel, diff, drupal_helpers, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m entity_usage, environment_indicator, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m fast_404, field_group, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m generated_content, gin, linkit, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m menu_trail_by_path, metatag, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m navigation_extra_tools, pathauto, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m redirect, reroute_email, robotstxt, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m "] +[44.647455, "o", " "] +[44.647536, "o", " \u001b[90m│\u001b[0m scheduled_transitions, schema_metatag, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m sdc_devel, seckit, shield, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m stage_file_proxy, testmode, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m views_bulk_operations, xmlsitemap \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Webroot \u001b[90m│\u001b[0m web \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Profile \u001b[90m│\u001b[0m standard \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Module prefix \u001b[90m│\u001b[0m sw \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Custom modules \u001b[90m│\u001b[0m base, search, demo \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Theme machine name "] +[44.647557, "o", " "] +[44.647648, "o", " \u001b[90m│\u001b[0m star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Build front-end in container \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Storybook component library \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mCode repository\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Code provider "] +[44.647654, "o", " \u001b[90m"] +[44.647758, "o", "│\u001b[0m github \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Version scheme \u001b[90m│\u001b[0m calver \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mEnvironment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Timezone \u001b[90m│\u001b[0m UTC \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Services \u001b[90m│\u001b["] +[44.647784, "o", "0m cla"] +[44.647867, "o", "mav, redis, solr \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Tools \u001b[90m│\u001b[0m behat, dclint, eslint, hadolint, jest, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m phpcs, phpstan, phpunit, rector, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m stylelint, twig_cs_fixer \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mHosting\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────"] +[44.647893, "o", "──"] +[44.648018, "o", "──────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Hosting provider \u001b[90m│\u001b[0m none \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDeployment\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Deployment types \u001b[90m│\u001b[0m webhook \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├───────────────────────"] +[44.648044, "o", "──"] +[44.648135, "o", "─────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mWorkflow\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Provision type \u001b[90m│\u001b[0m database \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Database source \u001b[90m│\u001b[0m url \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Database store \u001b[90m│\u001b[0m none \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Migration database \u001b[90m│\u001b[0m No "] +[44.648141, "o", " "] +[44.648264, "o", " \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mNotifications\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Channels \u001b[90m│\u001b[0m email \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴──────────────────────────────────────"] +[44.648291, "o", "─"] +[44.648372, "o", "─┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mContinuous Integration\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m CI provider \u001b[90m│\u001b[0m gha \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Visual regression testing \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Lighthouse audits \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Secret scanning with Gitleaks \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├─────────────────────────────────"] +[44.648416, "o", "─┴─"] +[44.6485, "o", "───────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mAutomations\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Dependency updates provider \u001b[90m│\u001b[0m renovatebot_app \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Auto-merge dependency updates \u001b[90m│\u001b[0m php, drupal_core, js, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m container_images, ci_tools, \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m github_actions \u001b[90m│\u001b[0m\r\r\n\u001b[90m│"] +[44.648506, "o", "\u001b[0m \u001b["] +[44.648626, "o", "90m│\u001b[0m Code coverage provider \u001b[90m│\u001b[0m none \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Auto-assign PR author \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Auto-add a CONFLICT label to PRs \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mDocumentation\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m"] +[44.648636, "o", "│\u001b[0m "] +[44.64872, "o", "Preserve project documentation \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mAI\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m AI agent instructions \u001b[90m│\u001b[0m Yes \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m AI code review \u001b[90m│\u001b[0m No \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├───"] +[44.648735, "o", "──"] +[44.648823, "o", "─────────────────────────────┴────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mLocations\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m├──────────────────────────────────┬────────────────────────────────────────┤\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Current directory \u001b[90m│\u001b[0m /home/user/vortex/.artifa \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m cts/tmp/videos-workspace \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Destination directory \u001b[90m│\u001b[0m /home/user/vortex/.artifa \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m "] +[44.648858, "o", " "] +[44.648945, "o", " \u001b[90m│\u001b[0m cts/tmp/videos-workspace/star_wars \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Vortex repository \u001b[90m│\u001b[0m /home/user/vortex \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Vortex reference \u001b[90m│\u001b[0m HEAD \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└──────────────────────────────────┴────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[44.652366, "o", " Vortex will be installed into your project's directory \"/home/user/demo/star_wars\"\r\r\n"] +[44.667097, "o", "\u001b[?25l"] +[44.667175, "o", "\u001b[36m◇\u001b[0m Proceed with installing Vortex?\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m●\u001b[0m Yes \u001b[2m/\u001b[0m \u001b[2m○\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[46.671218, "o", "\u001b[4A\r\u001b[J\u001b[36m◆\u001b[0m Proceed with installing Vortex?\r\r\n\u001b[90m│\u001b[0m \u001b[2mYes\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[46.679072, "o", "\u001b[?25h\u001b[34m●\u001b[0m \u001b[34mStarting project installation\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[46.679166, "o", "\u001b[36m◇\u001b[0m Downloading Vortex\r\r\n"] +[46.872093, "o", "\u001b[90m│\u001b[0m \u001b[2mDownloading from \"/home/user/vortex\" repository at ref \"HEAD\"\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Vortex downloaded (develop)\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[46.872182, "o", "\u001b[36m◇\u001b[0m Customizing Vortex for your project\r\r\n"] +[47.863451, "o", "\u001b[90m│\u001b[0m \u001b[32m✓ Vortex was customized for your project\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[47.863513, "o", "\u001b[36m◇\u001b[0m Preparing destination directory\r\r\n"] +[47.877398, "o", "\u001b[90m│\u001b[0m \u001b[2mCreated directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mInitializing a new Git repository in directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Destination directory is ready\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[47.877523, "o", "\u001b[36m◇\u001b[0m Copying files to the destination directory\r\r\n"] +[48.054067, "o", "\u001b[90m│\u001b[0m \u001b[32m✓ Files copied to destination directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[48.054153, "o", "\u001b[36m◇\u001b[0m Preparing demo content\r\r\n"] +[48.746067, "o", "\u001b[90m│\u001b[0m \u001b[2mCreated data directory\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars/.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdata\".\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo database dump file was found in\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m\"/home/user/demo/star_wars/.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mdata\" directory.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mFetched demo database from\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mhttps://github.com/drevops/vortex/releases/download/1.40.0/db.demo.sql.\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[32m✓ Demo content prepared\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[48.747772, "o", "\u001b[90m│\u001b[0m \u001b[90m┌────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mFinished installing Vortex\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m──────────────────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Add and commit all files: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m git add -A \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m git commit -m \"Initial commit.\" \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Files: 402 added \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Status and diff of every file are recorded in: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m .logs/vortex-install-2026-11-16-090813"] +[48.747781, "o", ".md \u001b["] +[48.747922, "o", "90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[48.764024, "o", "\u001b[?25l"] +[48.76421, "o", "\u001b[36m◇\u001b[0m Run the site build now?\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2m○\u001b[0m \u001b[2mYes\u001b[0m \u001b[2m/\u001b[0m \u001b[32m●\u001b[0m No\r\r\n\u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mTakes ~5-10 min; output will be streamed. You can skip and run later with:\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[2mahoy build\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[50.768891, "o", "\u001b[7A\r\u001b[J\u001b[36m◆\u001b[0m Run the site build now?\r\r\n\u001b[90m│\u001b[0m \u001b[2mNo\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[50.777504, "o", "\u001b[?25h"] +[50.778, "o", "\u001b[90m│\u001b[0m \u001b[90m┌───────────────────────────────────────────────────────────────────────────┐\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[1mReady to build\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m──────────────\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Build the site: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m ahoy build \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m Setup GitHub Actions: \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m"] +[50.778014, "o", " "] +[50.778103, "o", " \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m https://www.vortextemplate.com/docs/continuous-integration/github-actio \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m ns#onboarding \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m│\u001b[0m \u001b[90m│\u001b[0m\r\r\n\u001b[90m│\u001b[0m \u001b[90m└───────────────────────────────────────────────────────────────────────────┘\u001b[0m\r\r\n\u001b[90m│\u001b[0m\r\r\n"] +[50.788272, "x", "0"] diff --git a/.vortex/docs/static/img/installer.svg b/.vortex/docs/static/img/installer.svg index fc88e3393a..56e6d49256 100644 --- a/.vortex/docs/static/img/installer.svg +++ b/.vortex/docs/static/img/installer.svg @@ -1 +1 @@ -──────────────────────────────────────────────────────────────────────────────██╗██╗██████╗██████╗████████╗███████╗██╗██╗██║██║██╔═══██╗██╔══██╗╚══██╔══╝██╔════╝╚██╗██╔╝██║██║██║██║██████╔╝██║█████╗╚███╔╝╚██╗██╔╝██║██║██╔══██╗██║██╔══╝██╔██╗╚████╔╝╚██████╔╝██║██║██║███████╗██╔╝██╗╚═══╝╚═════╝╚═╝╚═╝╚═╝╚══════╝╚═╝╚═╝DrupalprojecttemplatebyDrevOpsInstallerversion:development│┌───────────────────────────────────────────────────────────────────────────┐││WelcometotheVortexinteractiveinstaller│││───────────────────────────────────────────│││Thistoolwillguideyouthroughinstallingthelatestdevelopment│││versionofVortexintoyourproject.││││││Youwillbeaskedafewquestionstotailortheconfigurationtoyour│││site.│││Nochangeswillbemadeuntilyouconfirmeverythingattheend.│││PressCtrl+Catanytimetoexittheinstaller.│││PressCtrl+Uatanytimetogobacktothepreviousstep.││└───────────────────────────────────────────────────────────────────────────┘│◇Validatingrepositoryandreference│Checkingrepository"/home/user/vortex"andreference"HEAD"│✓Repositoryandreferencevalidatedsuccessfully┌Generalinformation◇Sitename(1/33)│Wewillusethisnameintheprojectanddocumentation.│star█│E.g.MySite█│Wewillu│Sta█│StarWar█◆Sitename(1/33)│StarWars◆Sitemachinename(2/33)│star_wars◇Organizationname(3/33)│E.g.MyOrg█│E.g.MyO│R█│Rebell█◆Organizationname(3/33)│Rebellion◆Organizationmachinename(4/33)│rebellion◆Publicdomain(5/33)│star-wars.com├Drupal◇Howwouldyoulikeyoursitetobecreatedonthefirstrun?(6/33)│Choosehowyoursitewillbecreatedthefirsttimeafterthisinstaller│finishes:│○Drupal,installedfromprofile│Createsanewsitebypopulatingafreshdatabase│fromoneofthestandardDrupalinstallationprofiles.│○DrupalCMS,installedfromprofile◆Howwouldyoulikeyoursitetobecreatedonthefirstrun?(6/33)│Drupal,loadedfromthedemodatabase◆Profile(7/33)│Standard◇Modules(8/33)│❯◼Coffee│◼Configsplit│◼Configupdate│◼Devel│◼Diff│◼Drupalhelpers│◼Entityusage│◼Environmentindicator◆Modules(8/33)│Coffee,Configsplit,Configupdate,Devel,Diff,Drupalhelpers,Entity│usage,Environmentindicator,Fast404,Fieldgroup,Generatedcontent,Gin│admintheme,Linkit,Menutrailbypath,Metatag,Navigationextratools,│Pathauto,Redirect,Rerouteemail,Robots.txt,Scheduledtransitions,│Schema.orgMetatag,SDCDevel,Seckit,Shield,Stagefileproxy,Testmode,│Viewsbulkoperations,XMLSitemap◆Custommodulesprefix(9/33)│sw◇Custommodules(10/33)│Selectwhichcustommodulestoincludeinyourproject:│○Base│Startermodulewithdeployhooksandtestscaffoldingfor│Unit,Kernel,Functional,andFunctionalJavascripttests.│○Search│CustomSolrsearchintegrationmodule.RequirestheSolr│servicetobeselected.│○Demo│DemonstrateshowVortextoolingworks:includesacounter│blockwithCSS/JS,PHPUnitexampletestsacrossalltest│types,andaBehatfeature.Safetoremoveonrealprojects.│❯◼Base-startermodulewithutilitiesandtestscaffolding│◼Search-customSolrsearchintegration◆Custommodules(10/33)│Base-startermodulewithutilitiesandtestscaffolding,Search-custom│Solrsearchintegration,Demo-counterblockandexampleteststo│demonstratetooling◆Theme(11/33)│Custom(nextprompt)│││◇Customthememachinename(11.1)│◆Customthememachinename(11.1)││star_wars│◆Buildfront-endassetsinthecontainer?(11.2)││Yes│◇Storybookcomponentlibrary?(11.3)│Componentlibraryforthecustomtheme,renderedbyDrupalitself.│StoriesareauthoredinTwignexttoeachcomponent,sothelibraryshows│thesamemarkupthesiterenders.│◆Storybookcomponentlibrary?(11.3)│No├Coderepository◆Repositoryprovider(12/33)│GitHub◇Releaseversioningscheme(13/33)│Chooseyourversioningscheme:│○CalendarVersioning(CalVer)│year.month.patch(E.g.,24.1.0)│https://calver.org│○Other◆Releaseversioningscheme(13/33)│CalendarVersioning(CalVer)├Environment◆Timezone(14/33)│UTC◆Services(15/33)│ClamAV,Redis,Solr◇Developmenttools(16/33)◆Developmenttools(16/33)│Behat,DCLint,ESLint,Hadolint,Jest,PHPCodeSniffer,PHPStan,PHPUnit,│Rector,Stylelint,TwigCSFixer├Hosting│❯●None◆Hostingprovider(17/33)│None◆Customwebrootdirectory(18/33)│web├Deployment◆Deploymenttypes(19/33)│Customwebhook├Workflow◇Provisiontype(20/33)│Provisioningsetsupthesiteinanenvironmentusinganalreadyassembled│codebase.│○Importfromdatabasedump│Provisionsthesitebyimportingadatabasedump│typicallycopiedfromproductionintolower│environments.│○Installfromprofile│ProvisionsthesitebyinstallingafreshDrupal│sitefromaprofileeverytimeanenvironmentis│created.│❯●Importfromdatabasedump│○Installfromprofile◆Provisiontype(20/33)│Importfromdatabasedump│◆Databasesource(20.1)│URLdownload│○Yes/●No◆Useaseconddatabaseformigrations?(21/33)│No├Notifications◆Notificationchannels(22/33)│Email├ContinuousIntegration│○None◆ContinuousIntegrationprovider(23/33)│GitHubActions◆VisualregressiontestingwithDiffy?(24/33)◆PerformanceandSEOauditswithLighthouse?(25/33)│●Yes/○No◆ScanforcommittedsecretswithGitleaks?(26/33)│Yes├Automations◆Dependencyupdatesprovider(27/33)│RenovateGitHubapp│◇Auto-mergedependencyupdates(27.1)│Pullrequestsintheselectedgroupsaremergedassoonastheymeetthe│branchprotectionrules.Pullrequestsinothergroupsneedamanual│merge.│Requires"Allowauto-merge"intherepositorysettingsandatleastone│requiredstatuscheckonthebasebranch.│❯◼PHPpackages,exceptDrupalcore(minorandpatch)│◼Drupalcore(minorandpatch)│◼JavaScriptpackages(minorandpatch)│◼Containerimages(major,minorandpatch)│◼CItools(major,minorandpatch)│◼GitHubActions(major,minorandpatch)│◆Auto-mergedependencyupdates(27.1)│PHPpackages,exceptDrupalcore(minorandpatch),Drupalcore(minorand│patch),JavaScriptpackages(minorandpatch),Containerimages(major,│minorandpatch),CItools(major,minorandpatch),GitHubActions│(major,minorandpatch)◆Codecoverageprovider(28/33)◆Auto-assigntheauthortotheirPR?(29/33)◆Auto-addaCONFLICTlabeltoaPRwhenconflictsoccur?(30/33)├Documentation◆Preserveprojectdocumentation?(31/33)├AI◆ProvideAIagentinstructions?(32/33)◆ConfigureAIcodereviewwithCodeRabbit?(33/33)●Installationsummary││Generalinformation││├──────────────────────────────────┬────────────────────────────────────────┤││Sitename│StarWars│││Sitemachinename│star_wars│││Organizationname│Rebellion│││Organizationmachinename│rebellion│││Publicdomain│star-wars.com││├──────────────────────────────────┴────────────────────────────────────────┤││Drupal│││Starter│load_demodb│││Modules│coffee,config_split,config_update,││││devel,diff,drupal_helpers,││││entity_usage,environment_indicator,││││fast_404,field_group,││││generated_content,gin,linkit,││││menu_trail_by_path,metatag,││││navigation_extra_tools,pathauto,││││redirect,reroute_email,robotstxt,││││scheduled_transitions,schema_metatag,││││sdc_devel,seckit,shield,││││stage_file_proxy,testmode,││││views_bulk_operations,xmlsitemap│││Webroot│web│││Profile│standard│││Moduleprefix│sw│││Custommodules│base,search,demo│││Thememachinename││Thememachinename│star_wars│││Buildfront-endincontainer│Yes│││Storybookcomponentlibrary│No│││Coderepository│││Codeprovider││Codeprovider│github│││Versionscheme│calver│││Environment│││Timezone│UTC│││Services│clamav,redis,solr│││Tools│behat,dclint,eslint,hadolint,jest,││││phpcs,phpstan,phpunit,rector,││││stylelint,twig_cs_fixer│││Hosting│││Hostingprovider│none│││Deployment│││Deploymenttypes│webhook│││Workflow│││Provisiontype│database│││Databasesource│url│││Migrationdatabase│No│││Notifications│││Channels│email│││ContinuousIntegration││ContinuousIntegration│││CIprovider│gha│││Visualregressiontesting│No│││Lighthouseaudits│No│││SecretscanningwithGitleaks│Yes│││Automations│││Dependencyupdatesprovider│renovatebot_app│││Auto-mergedependencyupdates│php,drupal_core,js,││││container_images,ci_tools,││││github_actions│││Codecoverageprovider│none│││Auto-assignPRauthor│Yes│││Auto-addaCONFLICTlabeltoPRs│Yes│││Documentation│││Preserveprojectdocumentation│Yes│││AI│││AIagentinstructions│Yes│││AIcodereview│No│││Locations│││Currentdirectory│/home/user/vortex/.artifa││││cts/tmp/videos-workspace│││Destinationdirectory│/home/user/vortex/.artifa││││cts/tmp/videos-workspace/star_wars│││Vortexrepository│/home/user/vortex│││Vortexreference│HEAD││└──────────────────────────────────┴────────────────────────────────────────┘Vortexwillbeinstalledintoyourproject'sdirectory"/home/user/demo/star_wars"◆ProceedwithinstallingVortex?●Startingprojectinstallation◇DownloadingVortex│Downloadingfrom"/home/user/vortex"repositoryatref"HEAD"│✓Vortexdownloaded(develop)◇CustomizingVortexforyourproject│✓Vortexwascustomizedforyourproject◇Preparingdestinationdirectory│Createddirectory│"/home/user/demo/star_wars".│InitializinganewGitrepositoryindirectory│✓Destinationdirectoryisready◇Copyingfilestothedestinationdirectory│✓Filescopiedtodestinationdirectory◇Preparingdemocontent│Createddatadirectory│"/home/user/demo/star_wars/.│data".│Nodatabasedumpfilewasfoundin│data"directory.│Fetcheddemodatabasefrom│https://github.com/drevops/vortex/releases/download/1.40.0/db.demo.sql.│✓Democontentprepared│┌────────────────────────────────────────────────┐││FinishedinstallingVortex│││──────────────────────────│││Addandcommitallfiles:│││gitadd-A│││gitcommit-m"Initialcommit."││││││Files:402added│││Statusanddiffofeveryfilearerecordedin:│││.logs/vortex-install-2026-11-16-070622.md││└────────────────────────────────────────────────┘◆Runthesitebuildnow?││Readytobuild│││──────────────│││Buildthesite:│││ahoybuild│││SetupGitHubActions:│╚████╔╝╚██████╔╝██║██║██║╚████╔╝╚██████╔╝██║██║██║█││Youwillbeaskedafewquestionstotailorthec││Youwillbeaskedafewquestionstotailortheconfigu◇Pressanykeytocontinue...│starwars█│Wewilluse│Wewillusethisnameintheprojectan│We│S█│St█│Star█│Star█│StarW█│StarWa█│StarWars█◇Sitemachinename(2/33)│star_wars█│Wewillusethisnamefortheprojectdirectoryandinthecode.│StarWarsOrg█│StarWar◇O│R◆Organization│Createsanew│Createsanewsitebypopulat│fromtheDrupalCMSrecipe.│○Drupal,loadedfromthedemodatabase│Createsasitebyloadinganexistingdemodatabase│providedwiththeinstaller.│○Drupal,installedfromprofile│○DrupalCMS,installedfromprofile│❯●Drupal,loadedfromthedemodatabase│Use⬆and⬇.Appliesonlyonthefirstrunoftheinstaller.│◼│◼Fast404│◼Fieldgroup│↓19more│Use⬆,⬇andSpacebartoselectoneormoremodules.│◼Demo││star_wars█││Wewillusethisnameasacustomthemename│thesamemarkupthesite│Adevelopmentserverrunsondemandwith`ahoystorybook`.Astaticbuild│isproducedduringprovisioningandservedat`/storybook`inlocal,CI│anddevelopmentenvironments.│○Yes/●No│AddsStorybookpackagestothethemeandasteptoprovisioning.│○SemanticVersioning(SemVer)│major.minor.patch(E.g.,1.0.0)│https://semver.org│○Other│Customversioningschemeofyourchoice.│❯●CalendarVersioning(CalVer)│○SemanticVersioning(SemVer)│Use⬆and⬇toselectyourversionscheme.│❯◼PHPC│❯◼PHPCodeSniffer│◼PHPStan│◼Rector│◼ESLint│◼Stylelint│◼Jest│◼PHPUnit│◼Behat│◼TwigCSFixer│◼DCLint│↓1more│Use⬆,⬇andSpacebartoselectoneormoretools.◇Hostingprovider(17/33)│○AcquiaCloud│○Lagoon│Use⬆,⬇andSpacebartoselectyourhostingprovider.◇Customwebrootdirectory(18/33)│web█│Customdirectorywherethewebserverservesthesite.◇Deploymenttypes(19/33)│❯◻Codeartifact│◻Lagoonwebhook│◼Customwebhook│Use⬆,⬇andSpacebartoselectoneormoredeploymenttypes.│Use⬆and⬇│Use⬆and⬇toselecttheprovi│◇Databasesource(20.1)│❯●URLdownload│○FTPdownload│○Acquiabackup│○Lagoonenvironment│○Containerregistry│○S3bucket│○None│Use⬆and⬇toselectthedatabasefetchsource.◇Useaseconddatabaseformigrations?(21/33)│AddsaseconddatabaseserviceforDrupalmigrations.◇Notificationchannels(22/33)│❯◼Email│◻GitHub│◻JIRA│◻NewRelic│◻Slack│◻Webhook│Use⬆,⬇andSpacebartoselectoneormorenotificationchannels.◇ContinuousIntegrationprovider(23/33)│❯●GitHubActions│○CircleCI│Use⬆and⬇toselecttheCIprovider.◇VisualregressiontestingwithDiffy?(24/33)│Diffy-poweredvisualregressionworkflowtocomparedeployedenvironments.│Automaticallytriggerscomparisonsfordependency-updatepullrequests│matchingtheconfiguredbranchpattern(`deps/*`bydefault).│Runscomparisonsforpullrequeststaggedwiththe`VR`labelandpostsa│stickycomment.│AlsosupportsmanualrunsagainstanyURL.│RequiresaDiffyaccount.◇PerformanceandSEOauditswithLighthouse?(25/33)│Lighthouseauditsofperformance,accessibility,bestpracticesandSEOfor│thepagesyoulistorthepagesinthesitemap.│Runslocallywith`ahoytest-lighthouse`andinCIafterthetests,writing│HTMLandJSONreportsforeverypage.│Scorescanbecomparedwithper-categorybudgets.TheCIstepisadvisory│untilyoumakeitblocking.│AddsaLighthousecontainer,anAhoycommandandaCIstep.◇ScanforcommittedsecretswithGitleaks?(26/33)│RunsinCItodetecthardcodedpasswords,APIkeysandtokens.◇Dependencyupdatesprovider(27/33)│❯●RenovateGitHubapp│○Renovateself-hostedinCI│Use⬆and⬇toselectthedependencyupdatesprovider.│Use⬆,⬇an│Use⬆,⬇andSpacebartoselectoneormoregroups.◇Codecoverageprovider(28/33)│○Codecov│Use⬆and⬇toselectthecodecoverageprovider.◇Auto-assigntheauthortotheirPR?(29/33)│HelpstokeepthePRsorganized.◇Auto-addaCONFLICTlabeltoaPRwhenconflictsoccur?(30/33)│HelpstokeepquicklyidentifyPRsthatneedattention.◇Preserveprojectdocumentation?(31/33)│Helpstomaintaintheprojectdocumentationwithintherepository.◇ProvideAIagentinstructions?(32/33)│ProvidesAIcodingagentswithbettercontextabouttheproject.◇ConfigureAIcodereviewwithCodeRabbit?(33/33)│RequirestheCodeRabbitappontherepository.││Services│││Services│cla│├──────────────────────────────────┬────────────────────────────│├──────────────────────────────────┬──────────────────────────────│├───────────────────────│├─────────────────────────│├──────────────────────────│├─────────────────────────────│├──────────────────────────────────┴────────────────────────────────────────│├─│├──────────────────────────────────┴──────────│├──────────────────────────────────┴────────────││Vortexreposit││Vortexrepository◇ProceedwithinstallingVortex?││.logs/vortex-install-2026-11-16-070622││.logs/vortex-install-2026-11-16-070622.md◇Runthesitebuildnow?│Takes~5-10min;outputwillbestreamed.Youcanskipandrunlaterwith:│ahoybuild││https://www.vortextemplate.com/docs/continuous-integration/github-actio│││ns#onboarding│ \ No newline at end of file +──────────────────────────────────────────────────────────────────────────────██╗██╗██████╗██████╗████████╗███████╗██╗██╗██║██║██╔═══██╗██╔══██╗╚══██╔══╝██╔════╝╚██╗██╔╝██║██║██║██║██████╔╝██║█████╗╚███╔╝╚██╗██╔╝██║██║██╔══██╗██║██╔══╝██╔██╗╚████╔╝╚██████╔╝██║██║██║███████╗██╔╝██╗╚═══╝╚═════╝╚═╝╚═╝╚═╝╚══════╝╚═╝╚═╝DrupalprojecttemplatebyDrevOpsInstallerversion:development│┌───────────────────────────────────────────────────────────────────────────┐││WelcometotheVortexinteractiveinstaller│││───────────────────────────────────────────│││Thistoolwillguideyouthroughinstallingthelatestdevelopment│││versionofVortexintoyourproject.││││││Youwillbeaskedafewquestionstotailortheconfigurationtoyour│││site.│││Nochangeswillbemadeuntilyouconfirmeverythingattheend.│││PressCtrl+Catanytimetoexittheinstaller.│││PressCtrl+Uatanytimetogobacktothepreviousstep.││└───────────────────────────────────────────────────────────────────────────┘│◇Validatingrepositoryandreference│Checkingrepository"/home/user/vortex"andreference"HEAD"│✓Repositoryandreferencevalidatedsuccessfully┌Generalinformation◇Sitename(1/33)│Wewillusethisnameintheprojectanddocumentation.│star█│E.g.MySite█│We│StarWar█◆Sitename(1/33)│StarWars◆Sitemachinename(2/33)│star_wars◇Organizationname(3/33)◇O│E.g.MyOrg█│E.g.MyO│Rebe█│Rebellion█◆Organizationname(3/33)│Rebellion◆Organizationmachinename(4/33)│rebellion◆Publicdomain(5/33)│star-wars.com├Drupal◇Howwouldyoulikeyoursitetobecreatedonthefirstrun?(6/33)│Choosehowyoursitewillbecreatedthefirsttimeafterthisinstaller│finishes:│○Drupal,installedfromprofile│Createsanewsitebypopulatingafreshdatabase│fromoneofthestandardDrupalinstallationprofiles.│○DrupalCMS,installedfromprofile│fromtheDrupalCMSrecipe.│○Drupal,loadedfromthedemodatabase│Createsasitebyloadinganexistingdemodatabase│providedwiththeinstaller.│○Drupal,installedfromprofile│○DrupalCMS,installedfromprofile│❯●Drupal,loadedfromthedemodatabase◆Howwouldyoulikeyoursitetobecreatedonthefirstrun?(6/33)│Drupal,loadedfromthedemodatabase◆Profile(7/33)│Standard◇Modules(8/33)│❯◼Coffee│◼Configsplit│◼Configupdate│◼Devel│◼Diff│◼Drupalhelpers◆Modules(8/33)│Coffee,Configsplit,Configupdate,Devel,Diff,Drupalhelpers,Entity│usage,Environmentindicator,Fast404,Fieldgroup,Generatedcontent,Gin│admintheme,Linkit,Menutrailbypath,Metatag,Navigationextratools,│Pathauto,Redirect,Rerouteemail,Robots.txt,Scheduledtransitions,│Schema.orgMetatag,SDCDevel,Seckit,Shield,Stagefileproxy,Testmode,│Viewsbulkoperations,XMLSitemap◆Custommodulesprefix(9/33)│sw◇Custommodules(10/33)│Selectwhichcustommodulestoincludeinyourproject:│○Base│Startermodulewithdeployhooksandtestscaffoldingfor│Unit,Kernel,Functional,andFunctionalJavascripttests.│○Search│CustomSolrsearchintegrationmodule.RequirestheSolr│servicetobeselected.│○Demo│DemonstrateshowVortextoolingworks:includesacounter│blockwithCSS/JS,PHPUnitexampletestsacrossalltest│types,andaBehatfeature.Safetoremoveonrealprojects.│❯◼Base-startermodulewithutilitiesandtestscaffolding│◼Search-customSolrsearchintegration◆Custommodules(10/33)│Base-startermodulewithutilitiesandtestscaffolding,Search-custom│Solrsearchintegration,Demo-counterblockandexampleteststo│demonstratetooling◆Theme(11/33)│Custom(nextprompt)│││◇Customthememachinename(11.1)│◆Customthememachinename(11.1)││star_wars│◆Buildfront-endassetsinthecontainer?(11.2)││Yes│◇Storybookcomponentlibrary?(11.3)│Componentlibraryforthecustomtheme,renderedbyDrupalitself.│StoriesareauthoredinTwignexttoeachcomponent,sothelibraryshows│thesamemarkupthesiterenders.│◆Storybookcomponentlibrary?(11.3)│No├Coderepository◆Repositoryprovider(12/33)│GitHub◇Releaseversioningscheme(13/33)│Chooseyourversioningscheme:│○CalendarVersioning(CalVer)│year.month.patch(E.g.,24.1.0)│https://calver.org│○Other◆Releaseversioningscheme(13/33)│CalendarVersioning(CalVer)├Environment│UTC◆Timezone(14/33)◆Services(15/33)│ClamAV,Redis,Solr◆Developmenttools(16/33)│Behat,DCLint,ESLint,Hadolint,Jest,PHPCodeSniffer,PHPStan,PHPUnit,│Rector,Stylelint,TwigCSFixer├Hosting│❯●None◆Hostingprovider(17/33)│None◆Customwebrootdirectory(18/33)│web├Deployment◆Deploymenttypes(19/33)│Customwebhook├Workflow◇Provisiontype(20/33)│Provisioningsetsupthesiteinanenvironmentusinganalreadyassembled│codebase.│○Importfromdatabasedump│Provisionsthesitebyimportingadatabasedump│typicallycopiedfromproductionintolower│environments.│○Installfromprofile│ProvisionsthesitebyinstallingafreshDrupal│sitefromaprofileeverytimeanenvironmentis│created.│❯●Importfromdatabasedump│○Installfromprofile◆Provisiontype(20/33)│Importfromdatabasedump│◆Databasesource(20.1)│URLdownload│││◆Databasestore(20.1.1)│None,fetchfromthesource│○Yes/●No◆Useaseconddatabaseformigrations?(21/33)│No├Notifications◆Notificationchannels(22/33)│Email├ContinuousIntegration│○None◆ContinuousIntegrationprovider(23/33)│GitHubActions◆VisualregressiontestingwithDiffy?(24/33)◆PerformanceandSEOauditswithLighthouse?(25/33)│●Yes/○No◆ScanforcommittedsecretswithGitleaks?(26/33)│Yes├Automations◆Dependencyupdatesprovider(27/33)│RenovateGitHubapp│◇Auto-mergedependencyupdates(27.1)│Pullrequestsintheselectedgroupsaremergedassoonastheymeetthe│branchprotectionrules.Pullrequestsinothergroupsneedamanual│merge.│Requires"Allowauto-merge"intherepositorysettingsandatleastone│requiredstatuscheckonthebasebranch.│❯◼PHPpackages,exceptDrupalcore(minorandpatch)│◼Drupalcore(minorandpatch)│◼JavaScriptpackages(minorandpatch)│◼Containerimages(major,minorandpatch)│◆Auto-mergedependencyupdates(27.1)│PHPpackages,exceptDrupalcore(minorandpatch),Drupalcore(minorand│patch),JavaScriptpackages(minorandpatch),Containerimages(major,│minorandpatch),CItools(major,minorandpatch),GitHubActions│(major,minorandpatch)◆Codecoverageprovider(28/33)◆Auto-assigntheauthortotheirPR?(29/33)◆Auto-addaCONFLICTlabeltoaPRwhenconflictsoccur?(30/33)├Documentation◆Preserveprojectdocumentation?(31/33)├AI◆ProvideAIagentinstructions?(32/33)◆ConfigureAIcodereviewwithCodeRabbit?(33/33)●Installationsummary││Generalinformation││├──────────────────────────────────┬────────────────────────────────────────┤││Sitename│StarWars│││Sitemachinename│star_wars│││Organizationname│Rebellion│││Organizationmachinename│rebellion│││Publicdomain│star-wars.com││├──────────────────────────────────┴────────────────────────────────────────┤││Drupal│││Starter│load_demodb│││Modules│coffee,config_split,config_update,││││devel,diff,drupal_helpers,││││entity_usage,environment_indicator,││││fast_404,field_group,││││generated_content,gin,linkit,││││menu_trail_by_path,metatag,││││navigation_extra_tools,pathauto,││││redirect,reroute_email,robotstxt,││││scheduled_transitions,schema_metatag,││││sdc_devel,seckit,shield,││││stage_file_proxy,testmode,││││views_bulk_operations,xmlsitemap│││Webroot│web│││Profile│standard│││Moduleprefix│sw│││Custommodules│base,search,demo│││Thememachinename││Thememachinename│star_wars│││Buildfront-endincontainer│Yes│││Storybookcomponentlibrary│No│││Coderepository│││Codeprovider││Codeprovider│github│││Versionscheme│calver│││Environment│││Timezone│UTC│││Services│clamav,redis,solr│││Tools│behat,dclint,eslint,hadolint,jest,││││phpcs,phpstan,phpunit,rector,││││stylelint,twig_cs_fixer│││Hosting│││Hostingprovider│none│││Deployment│││Deploymenttypes│webhook│││Workflow│││Provisiontype│database│││Databasesource│url│││Databasestore│none│││Migrationdatabase│No││Migrationdatabase│No│││Notifications│││Channels│email│││ContinuousIntegration│││CIprovider│gha│││Visualregressiontesting│No│││Lighthouseaudits│No│││SecretscanningwithGitleaks│Yes│││Automations│││Dependencyupdatesprovider│renovatebot_app│││Auto-mergedependencyupdates│php,drupal_core,js,││││container_images,ci_tools,││││github_actions│││Codecoverageprovider│none│││Auto-assignPRauthor│Yes│││Auto-addaCONFLICTlabeltoPRs│Yes│││Documentation│││Preserveprojectdocumentation│Yes│││AI│││AIagentinstructions│Yes│││AIcodereview│No│││Locations│││Currentdirectory│/home/user/vortex/.artifa││││cts/tmp/videos-workspace│││Destinationdirectory│/home/user/vortex/.artifa││││cts/tmp/videos-workspace/star_wars│││Vortexrepository│/home/user/vortex│││Vortexreference│HEAD││└──────────────────────────────────┴────────────────────────────────────────┘Vortexwillbeinstalledintoyourproject'sdirectory"/home/user/demo/star_wars"◆ProceedwithinstallingVortex?●Startingprojectinstallation◇DownloadingVortex│Downloadingfrom"/home/user/vortex"repositoryatref"HEAD"│✓Vortexdownloaded(develop)◇CustomizingVortexforyourproject│✓Vortexwascustomizedforyourproject◇Preparingdestinationdirectory│Createddirectory│"/home/user/demo/star_wars".│InitializinganewGitrepositoryindirectory│✓Destinationdirectoryisready◇Copyingfilestothedestinationdirectory│✓Filescopiedtodestinationdirectory◇Preparingdemocontent│Createddatadirectory│"/home/user/demo/star_wars/.│data".│Nodatabasedumpfilewasfoundin│data"directory.│Fetcheddemodatabasefrom│https://github.com/drevops/vortex/releases/download/1.40.0/db.demo.sql.│✓Democontentprepared│┌────────────────────────────────────────────────┐││FinishedinstallingVortex│││──────────────────────────│││Addandcommitallfiles:│││gitadd-A│││gitcommit-m"Initialcommit."││││││Files:402added│││Statusanddiffofeveryfilearerecordedin:│││.logs/vortex-install-2026-11-16-090813.md││└────────────────────────────────────────────────┘◆Runthesitebuildnow?││Readytobuild│││──────────────│││Buildthesite:│││ahoybuild│││SetupGitHubActions:│╚████╔╝╚██████╔╝██║██║██║╚████╔╝╚██████╔╝██║██║██║█││Youwillbeaskedafewquestionstotailorthec││Youwillbeaskedafewquestionstotailortheconfigu◇Pressanykeytocontinue...│starwars█│Wewilluse│Wewillusethisnameintheprojectan│Wewillu│S█│St█│Sta█│Star█│Star█│StarW█│StarWa█│StarWars█◇Sitemachinename(2/33)│star_wars█│Wewillusethisnamefortheprojectdirectoryandinthecode.│StarWarsOrg█│StarWar│Wewillusethisnamei│Wewillusethisnameintheprojectanddocumentat│E.g.MyOrg│Wewillusethisnameinthepr│○│Us│◼Entityu│◼Entityusage│◼Environmentindicator│◼Fast404│◼Fieldgroup│↓19more│Use⬆,⬇andSpacebartoselectoneormoremodules.│◼Demo││star_wars█││Wewillusethisnameasacustomthemename│thesamemarkupthesite│Adevelopmentserverrunsondemandwith`ahoystorybook`.Astaticbuild│isproducedduringprovisioningandservedat`/storybook`inlocal,CI│anddevelopmentenvironments.│○Yes/●No│AddsStorybookpackagestothethemeandasteptoprovisioning.│○SemanticVersioning(SemVer)│major.minor.patch(E.g.,1.0.0)│https://semver.org│○Other│Customversioningschemeofyourchoice.│❯●CalendarVersioning(CalVer)│○SemanticVersioning(SemVer)│Use⬆and⬇toselectyourversionscheme.◇Timezone(14/33)│UTC█│Etc/UTC│Use⬆and⬇,orstarttypingtoselectthetimezoneforyourproject.◇Services(15/33)│❯◼ClamAV│◼Solr│◼Redis│Use⬆,⬇andSpacebartoselectoneormoreservices.◇Developmenttools(16/33)│❯◼PHPCodeSniffer│◼PHPStan│◼Rector│◼ESLint│◼Stylelint│◼Jest│◼PHPUnit│◼Behat│◼TwigCSFixer│◼DCLint│↓1more│Use⬆,⬇andSpacebartoselectoneormoretools.◇Hostingprovider(17/33)│○AcquiaCloud│○Lagoon│Use⬆,⬇andSpacebartoselectyourhostingprovider.◇Customwebrootdirectory(18/33)│web█│Customdirectorywherethewebserverservesthesite.◇Deploymenttypes(19/33)│❯◻Codeartifact│◻Lagoonwebhook│◼Customwebhook│Use⬆,⬇andSpacebartoselectoneormoredeploymenttypes.│Use⬆and⬇│Use⬆and⬇toselecttheprovi│Use⬆and⬇toselecttheprovisiontype.│◇Databasesource(20.1)│❯●URLdownload│○FTPdownload│○Acquiabackup│○Lagoonenvironment│○Containerregistry│○S3bucket│○None│Use⬆and⬇toselectthedatabasefetchsource.│◇Databasestore(20.1.1)│❯●None,fetchfromthesource│○FTPserver│○S3bucket│Use⬆and⬇toselectwheredevelopersfetchthesanitizeddatabase│from.◇Useaseconddatabaseformigrations?(21/33)│AddsaseconddatabaseserviceforDrupalmigrations.◇Notificationchannels(22/33)│❯◼Email│◻GitHub│◻JIRA│◻NewRelic│◻Slack│◻Webhook│Use⬆,⬇andSpacebartoselectoneormorenotificationchannels.◇ContinuousIntegrationprovider(23/33)│❯●GitHubActions│○CircleCI│Use⬆and⬇toselecttheCIprovider.◇VisualregressiontestingwithDiffy?(24/33)│Diffy-poweredvisualregressionworkflowtocomparedeployedenvironments.│Automaticallytriggerscomparisonsfordependency-updatepullrequests│matchingtheconfiguredbranchpattern(`deps/*`bydefault).│Runscomparisonsforpullrequeststaggedwiththe`VR`labelandpostsa│stickycomment.│AlsosupportsmanualrunsagainstanyURL.│RequiresaDiffyaccount.◇PerformanceandSEOauditswithLighthouse?(25/33)│Lighthouseauditsofperformance,accessibility,bestpracticesandSEOfor│thepagesyoulistorthepagesinthesitemap.│Runslocallywith`ahoytest-lighthouse`andinCIafterthetests,writing│HTMLandJSONreportsforeverypage.│Scorescanbecomparedwithper-categorybudgets.TheCIstepisadvisory│untilyoumakeitblocking.│AddsaLighthousecontainer,anAhoycommandandaCIstep.◇ScanforcommittedsecretswithGitleaks?(26/33)│RunsinCItodetecthardcodedpasswords,APIkeysandtokens.◇Dependencyupdatesprovider(27/33)│❯●RenovateGitHubapp│○Renovateself-hostedinCI│Use⬆and⬇toselectthedependencyupdatesprovider.│◼CI│◼CItools(major,mi│◼CItools(major,minorandpatch)│◼GitHubActions(major,minorandpatch)│Use⬆,⬇andSpacebartoselectoneormoregroups.◇Codecoverageprovider(28/33)│○Codecov│Use⬆and⬇toselectthecodecoverageprovider.◇Auto-assigntheauthortotheirPR?(29/33)│HelpstokeepthePRsorganized.◇Auto-addaCONFLICTlabeltoaPRwhenconflictsoccur?(30/33)│HelpstokeepquicklyidentifyPRsthatneedattention.◇Preserveprojectdocumentation?(31/33)│Helpstomaintaintheprojectdocumentationwithintherepository.◇ProvideAIagentinstructions?(32/33)│ProvidesAIcodingagentswithbettercontextabouttheproject.◇ConfigureAIcodereviewwithCodeRabbit?(33/33)│RequirestheCodeRabbitappontherepository.││Services│││Services│cla│├──────────────────────────────────┬────────────────────────────│├──────────────────────────────────┬──────────────────────────────│├───────────────────────│├─────────────────────────│├──────────────────────────────────┴──────────────────────────────────────│├──────────────────────────────────┴───────────────────────────────────────│├─────────────────────────────────│├──────────────────────────────────┴─│├───│├─────◇ProceedwithinstallingVortex?││.logs/vortex-install-2026-11-16-090813││.logs/vortex-install-2026-11-16-090813.md◇Runthesitebuildnow?│Takes~5-10min;outputwillbestreamed.Youcanskipandrunlaterwith:│ahoybuild││https://www.vortextemplate.com/docs/continuous-integration/github-actio│││ns#onboarding│ \ No newline at end of file