#!/usr/bin/env bash # Lab kit for the Metrics User course: send hand-made datapoints to Splunk Observability Cloud. # The token is read from a chmod 600 file and passed to curl on stdin, never on the command line. # Set REALM to your org's realm (default us1) and TOKEN_FILE if you keep the token elsewhere. # # ./send.sh test one datapoint, prints OK if accepted # ./send.sh mts [min] lesson 1-2: gauge lab.cpu.utilization, web-01 and web-02 (env=prod), every 10 s # ./send.sh region [min] lesson 1: same, but web-01 also sends region=us-east (a new dimension key) # ./send.sh types [min] lesson 1: counter lab.requests + cumulative counter lab.requests.total, restart halfway # ./send.sh analytics [min] lesson 3: cpu for web-01..04 in two envs plus web-05 with no env; cache hits and misses # ./send.sh detector-demo [min] lesson 4: normal, 3-minute spike, recovery, 1-point blip, web-02 goes silent 2 minutes # # Default run length is 5 minutes (detector-demo: 25). Stop any time with Ctrl-C. set -euo pipefail REALM="${REALM:-us1}" TOKEN_FILE="${TOKEN_FILE:-$HOME/.config/splunk-o11y/ingest-token}" URL="https://ingest.${REALM}.signalfx.com/v2/datapoint" [[ -r "$TOKEN_FILE" ]] || { echo "No token file at $TOKEN_FILE"; exit 1; } TOKEN="$(tr -d '[:space:]' < "$TOKEN_FILE")" [[ -n "$TOKEN" ]] || { echo "Token file is empty"; exit 1; } send() { local code code=$(printf 'X-SF-Token: %s\n' "$TOKEN" | curl -sS -o /dev/null -w '%{http_code}' -H @- -H 'Content-Type: application/json' \ --data-binary "$1" "$URL") || { echo "curl failed"; return 1; } [[ "$code" == 200 ]] || { echo "ingest returned HTTP $code"; return 1; } } cpu() { echo $(( RANDOM % 41 + 30 )); } # 30-70 rnd() { echo $(( RANDOM % ($2 - $1 + 1) + $1 )); } # rnd LOW HIGH g() { printf '{"metric":"%s","value":%s,"dimensions":{%s}}' "$1" "$2" "$3"; } mode="${1:-}" default=5; [[ "$mode" == detector-demo ]] && default=25 minutes="${2:-$default}" rounds=$(( minutes * 6 )) case "$mode" in test) send '{"gauge":[{"metric":"lab.hello","value":1,"dimensions":{"host":"web-01"}}]}' echo "OK: datapoint accepted by ${REALM}" ;; mts) for (( i = 1; i <= rounds; i++ )); do send "{\"gauge\":[ {\"metric\":\"lab.cpu.utilization\",\"value\":$(cpu),\"dimensions\":{\"host\":\"web-01\",\"env\":\"prod\"}}, {\"metric\":\"lab.cpu.utilization\",\"value\":$(cpu),\"dimensions\":{\"host\":\"web-02\",\"env\":\"prod\"}}]}" echo "round $i/$rounds sent (2 MTS)"; sleep 10 done ;; region) for (( i = 1; i <= rounds; i++ )); do send "{\"gauge\":[ {\"metric\":\"lab.cpu.utilization\",\"value\":$(cpu),\"dimensions\":{\"host\":\"web-01\",\"env\":\"prod\",\"region\":\"us-east\"}}, {\"metric\":\"lab.cpu.utilization\",\"value\":$(cpu),\"dimensions\":{\"host\":\"web-02\",\"env\":\"prod\"}}]}" echo "round $i/$rounds sent (web-01 now has region)"; sleep 10 done ;; types) total=18400 for (( i = 1; i <= rounds; i++ )); do reqs=$(( RANDOM % 121 + 80 )) # 80-200 requests in this 10 s if (( i == rounds / 2 )); then total=0; echo "simulated restart: cumulative counter resets"; fi total=$(( total + reqs )) send "{\"counter\":[{\"metric\":\"lab.requests\",\"value\":$reqs,\"dimensions\":{\"host\":\"web-01\"}}], \"cumulative_counter\":[{\"metric\":\"lab.requests.total\",\"value\":$total,\"dimensions\":{\"host\":\"web-01\"}}]}" echo "round $i/$rounds sent (counter $reqs, cumulative $total)"; sleep 10 done ;; analytics) for (( i = 1; i <= rounds; i++ )); do pts=() for h in web-01:prod web-02:prod web-03:staging web-04:staging; do host="${h%%:*}" env="${h##*:}" d="\"host\":\"${host}\",\"env\":\"${env}\"" if [[ "$host" == web-04 ]]; then v=$(rnd 60 95); else v=$(cpu); fi pts+=("$(g lab.cpu.utilization "$v" "$d")" "$(g lab.cache.hits "$(rnd 700 900)" "$d")" "$(g lab.cache.misses "$(rnd 100 300)" "$d")") done pts+=("$(g lab.cpu.utilization "$(cpu)" '"host":"web-05"')") body=$(IFS=,; echo "${pts[*]}") send "{\"gauge\":[${body}]}" echo "round $i/$rounds sent (5 hosts, cpu + cache hits/misses)"; sleep 10 done ;; detector-demo) # One cycle is 121 rounds (about 20 minutes): normal, spike, recovery, blip, normal, web-02 silent. for (( i = 1; i <= rounds; i++ )); do k=$(( (i - 1) % 121 )) a=$(cpu) b=$(cpu) phase="normal" send_b=1 if (( k >= 30 && k < 48 )); then a=$(rnd 92 98); phase="web-01 spike (sustained)"; fi if (( k == 78 )); then a=95; phase="web-01 blip (one datapoint)"; fi if (( k >= 109 )); then send_b=0; phase="web-02 silent"; fi body="$(g lab.cpu.utilization "$a" '"host":"web-01","env":"prod"')" (( send_b )) && body+=",$(g lab.cpu.utilization "$b" '"host":"web-02","env":"prod"')" send "{\"gauge\":[${body}]}" echo "round $i/$rounds: $phase (web-01 $a)"; sleep 10 done ;; *) sed -n '2,15p' "$0"; exit 1 ;; esac