diff --git a/cloud-init.sh.in b/cloud-init.sh.in index 9fe5e23656..ba9d6eae92 100644 --- a/cloud-init.sh.in +++ b/cloud-init.sh.in @@ -2,6 +2,14 @@ # This is the script for cloud-init, to run on a VM in unattended fashion. See run-benchmark.sh +# Make sure the machine terminates whatever happens to this script: a deadline +# that systemd enforces even if the script hangs (a stalled download or upload, +# or a background server holding the pipe of `timeout ... | tee` open) or is +# killed, and a trap for early exits. The `shutdown now` at the end replaces +# the deadline in the normal case. +shutdown -P "+$(( @timeout@ / 60 + 120 ))" +trap 'shutdown now' EXIT + # Cloud-init runs scripts as root with no HOME exported. Tools that follow # XDG-ish conventions (DuckDB extensions in ~/.duckdb, the GizmoSQL one-line # installer that runs `sh -u`, etc.) then fall over with messages like @@ -86,12 +94,12 @@ send_log() { local f="$1" size size=$(stat -c%s "$f" 2>/dev/null || echo 0) if [ "$size" -le 1000000 ]; then - curl ${RESULTS_URL} --data-binary @"$f" + curl --max-time 300 ${RESULTS_URL} --data-binary @"$f" else { head -c 100000 "$f" printf '\n...[%d bytes truncated]...\n' "$((size - 1000000))" tail -c 900000 "$f" - } | curl ${RESULTS_URL} --data-binary @- + } | curl --max-time 300 ${RESULTS_URL} --data-binary @- fi } send_log log diff --git a/hardware/cloud-init.sh b/hardware/cloud-init.sh index bbb0ba2d71..ba959b4832 100755 --- a/hardware/cloud-init.sh +++ b/hardware/cloud-init.sh @@ -2,6 +2,9 @@ # See the docs in clickhouse/cloud-init.sh +# Terminate even if the benchmark hangs. +shutdown -P +600 + BASE_URL='https://raw.githubusercontent.com/ClickHouse/ClickBench/main/hardware/' apt-get update -y diff --git a/hardware/cloud-init.sh.in b/hardware/cloud-init.sh.in index 3a698cde5a..c13713e684 100644 --- a/hardware/cloud-init.sh.in +++ b/hardware/cloud-init.sh.in @@ -11,6 +11,14 @@ # and mounted, and the data is placed there. Otherwise it stays on the root # EBS volume. +# Make sure the machine terminates whatever happens to this script: a deadline +# that systemd enforces even if the script hangs (a stalled download or upload, +# or a background server holding the pipe of `timeout ... | tee` open) or is +# killed, and a trap for early exits. The `shutdown now` at the end replaces +# the deadline in the normal case. +shutdown -P "+$(( @timeout@ / 60 + 120 ))" +trap 'shutdown now' EXIT + export HOME="${HOME:-/root}" export DEBIAN_FRONTEND=noninteractive apt-get update -y @@ -109,18 +117,18 @@ jq -s -c \ --argjson tags "$(printf '%s\n' cloud aws "${ARCH}" ${VENDOR} "${STORAGE_TAG}" | jq -R . | jq -s -c .)" \ '{kind: $kind, instance_type: $instance_type, version: $version, machine: $machine, comment: $comment, time: $time, tags: $tags, result: .}' \ - result.lines | tee result.json | curl "${RESULTS_URL}" --data-binary @- + result.lines | tee result.json | curl --max-time 300 "${RESULTS_URL}" --data-binary @- # Also send the (truncated) log for diagnostics, as the main benchmark does. send_log() { local f="$1" size size=$(stat -c%s "$f" 2>/dev/null || echo 0) if [ "$size" -le 1000000 ]; then - curl "${RESULTS_URL}" --data-binary @"$f" + curl --max-time 300 "${RESULTS_URL}" --data-binary @"$f" else { head -c 100000 "$f" printf '\n...[%d bytes truncated]...\n' "$((size - 1000000))" - tail -c 900000 "$f"; } | curl "${RESULTS_URL}" --data-binary @- + tail -c 900000 "$f"; } | curl --max-time 300 "${RESULTS_URL}" --data-binary @- fi } send_log "${LOG}" diff --git a/versions/cloud-init.sh.in b/versions/cloud-init.sh.in index 54a210ca77..5a1b277bc6 100644 --- a/versions/cloud-init.sh.in +++ b/versions/cloud-init.sh.in @@ -9,6 +9,14 @@ # bucket -> (build the image from source if this version has no published one) # -> run-version.sh -> POST the result JSON + log to the sink -> shutdown. +# Make sure the machine terminates whatever happens to this script: a deadline +# that systemd enforces even if the script hangs (a stalled download or upload, +# or a background server holding the pipe of `timeout ... | tee` open) or is +# killed, and a trap for early exits. The `shutdown now` at the end replaces +# the deadline in the normal case. +shutdown -P "+$(( @timeout@ / 60 + 120 ))" +trap 'shutdown now' EXIT + export HOME="${HOME:-/root}" export DEBIAN_FRONTEND=noninteractive apt-get update -y @@ -91,7 +99,7 @@ RESULTS_URL='https://play.clickhouse.com/?user=sink&query=INSERT+INTO+data+FORMA if [ -f "results/${VERSION}.json" ]; then jq -c --arg machine "${MACHINE}" --arg kind 'versions-benchmark' \ '. + {kind: $kind, machine: $machine}' "results/${VERSION}.json" \ - | curl "${RESULTS_URL}" --data-binary @- + | curl --max-time 300 "${RESULTS_URL}" --data-binary @- fi # Also send the (truncated) log for diagnostics, as the main benchmark does. @@ -99,11 +107,11 @@ send_log() { local f="$1" size size=$(stat -c%s "$f" 2>/dev/null || echo 0) if [ "$size" -le 1000000 ]; then - curl "${RESULTS_URL}" --data-binary @"$f" + curl --max-time 300 "${RESULTS_URL}" --data-binary @"$f" else { head -c 100000 "$f" printf '\n...[%d bytes truncated]...\n' "$((size - 1000000))" - tail -c 900000 "$f"; } | curl "${RESULTS_URL}" --data-binary @- + tail -c 900000 "$f"; } | curl --max-time 300 "${RESULTS_URL}" --data-binary @- fi } send_log log