577 lines
15 KiB
Bash
577 lines
15 KiB
Bash
#!/bin/bash
|
||
set -euo pipefail
|
||
|
||
BASE_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
source "$BASE_DIR/config.conf"
|
||
|
||
mkdir -p "$(dirname "$LOG_FILE")"
|
||
|
||
LOG_LEVEL="${LOG_LEVEL:-INFO}"
|
||
|
||
level_to_int() {
|
||
case "$1" in
|
||
DEBUG) echo 0 ;;
|
||
INFO) echo 1 ;;
|
||
WARN) echo 2 ;;
|
||
ERROR) echo 3 ;;
|
||
*) echo 1 ;;
|
||
esac
|
||
}
|
||
|
||
should_log() {
|
||
[ "$(level_to_int "$1")" -ge "$(level_to_int "$LOG_LEVEL")" ]
|
||
}
|
||
|
||
log() {
|
||
local level="$1"; shift
|
||
local msg="$*"
|
||
if should_log "$level"; then
|
||
echo "$(date '+%Y-%m-%d %H:%M:%S') | $level | $msg" | tee -a "$LOG_FILE"
|
||
fi
|
||
}
|
||
|
||
# Log begrenzen
|
||
if [ -f "$LOG_FILE" ]; then
|
||
line_count=$(wc -l < "$LOG_FILE")
|
||
if [ "$line_count" -gt 1000 ]; then
|
||
tail -n 1000 "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"
|
||
fi
|
||
fi
|
||
|
||
# =============================
|
||
# Helper
|
||
# =============================
|
||
|
||
is_excluded() {
|
||
local svc="$1"
|
||
for ex in "${EXCLUDE_SERVICES[@]}"; do
|
||
[[ "$svc" == "$ex" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
is_stack_excluded() {
|
||
local stack="$1"
|
||
for ex in "${EXCLUDE_STACKS[@]}"; do
|
||
[[ "$stack" == "$ex" ]] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
get_image() {
|
||
local svc="$1"
|
||
|
||
echo "$compose_json" \
|
||
| jq -r --arg svc "$svc" '.services[$svc].image // empty'
|
||
}
|
||
|
||
get_container_image_id() {
|
||
local svc="$1"
|
||
local cid
|
||
cid=$(docker compose ps -q "$svc" 2>/dev/null || true)
|
||
[ -z "$cid" ] && echo "" && return
|
||
docker inspect -f '{{.Image}}' "$cid" 2>/dev/null || echo ""
|
||
}
|
||
|
||
get_local_image_id() {
|
||
local image="$1"
|
||
docker image inspect -f '{{.Id}}' "$image" 2>/dev/null || echo ""
|
||
}
|
||
|
||
is_running() {
|
||
local svc="$1"
|
||
local cid
|
||
cid=$(docker compose ps -q "$svc" 2>/dev/null || true)
|
||
[ -z "$cid" ] && return 1
|
||
docker inspect -f '{{.State.Running}}' "$cid" 2>/dev/null | grep -q true
|
||
}
|
||
|
||
run_cmd() {
|
||
if [ "$UPDATE_DRY_RUN" = true ]; then
|
||
log DEBUG "[DRY RUN] $*"
|
||
else
|
||
eval "$@"
|
||
fi
|
||
}
|
||
|
||
send_ntfy() {
|
||
local msg="$1"
|
||
local prio="$2"
|
||
local image_url="${NTFY_IMAGE_URL:-}"
|
||
|
||
curl -s \
|
||
-H "Authorization: Bearer $NTFY_TOKEN" \
|
||
-H "Title: $NTFY_TITLE" \
|
||
-H "Priority: $prio" \
|
||
-H "Tags: $NTFY_TAGS" \
|
||
${image_url:+-H "Icon: $image_url"} \
|
||
-d "$msg" \
|
||
"$NTFY_URL" > /dev/null || true
|
||
}
|
||
|
||
get_docker_disk_usage() {
|
||
local total=0
|
||
while read -r size; do
|
||
num="${size//[!0-9.]/}"
|
||
num="${num:-0}"
|
||
if [[ "$size" == *GB ]]; then
|
||
total=$(bc <<< "$total + ($num * 1024)")
|
||
elif [[ "$size" == *MB ]]; then
|
||
total=$(bc <<< "$total + $num")
|
||
elif [[ "$size" == *kB ]]; then
|
||
total=$(bc <<< "$total + ($num / 1024)")
|
||
fi
|
||
done < <(docker system df --format '{{.Size}}' 2>/dev/null)
|
||
|
||
LC_NUMERIC=C printf "%.0f\n" "$total"
|
||
}
|
||
|
||
wait_for_healthy() {
|
||
local timeout="$1"
|
||
shift
|
||
local services=("$@")
|
||
|
||
local start
|
||
start=$(date +%s)
|
||
|
||
# 👉 Containerliste bestimmen
|
||
if [ ${#services[@]} -gt 0 ]; then
|
||
mapfile -t cids < <(docker compose ps -q "${services[@]}")
|
||
else
|
||
mapfile -t cids < <(docker compose ps -q)
|
||
fi
|
||
|
||
# 👉 keine Container → nichts zu tun
|
||
if [ ${#cids[@]} -eq 0 ]; then
|
||
log INFO " ℹ️ Keine Container gefunden → überspringe Healthcheck"
|
||
return 0
|
||
fi
|
||
|
||
# 👉 prüfen ob überhaupt Healthchecks existieren
|
||
local has_healthcheck=false
|
||
|
||
for cid in "${cids[@]}"; do
|
||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$cid" 2>/dev/null)
|
||
|
||
if [ -n "$health" ]; then
|
||
has_healthcheck=true
|
||
break
|
||
fi
|
||
done
|
||
|
||
# 👉 wenn keiner vorhanden → direkt raus
|
||
if [ "$has_healthcheck" = false ]; then
|
||
log INFO " ℹ️ Keine Healthchecks definiert → überspringe warten"
|
||
return 0
|
||
fi
|
||
|
||
log INFO " ⏳ Warte auf healthy Container (max ${timeout}s)"
|
||
|
||
while true; do
|
||
local all_ok=true
|
||
|
||
for cid in "${cids[@]}"; do
|
||
status=$(docker inspect -f '{{.State.Health.Status}}' "$cid" 2>/dev/null || echo "none")
|
||
|
||
if [ "$status" = "starting" ] || [ "$status" = "unhealthy" ]; then
|
||
all_ok=false
|
||
break
|
||
fi
|
||
done
|
||
|
||
if [ "$all_ok" = true ]; then
|
||
log INFO " ✔️ Alle Container healthy"
|
||
return 0
|
||
fi
|
||
|
||
local now
|
||
now=$(date +%s)
|
||
|
||
if [ $((now - start)) -ge "$timeout" ]; then
|
||
log WARN " ⚠️ Healthcheck Timeout erreicht"
|
||
return 1
|
||
fi
|
||
|
||
sleep 2
|
||
done
|
||
}
|
||
|
||
get_remote_digest() {
|
||
local image="$1"
|
||
|
||
docker manifest inspect "$image" 2>/dev/null \
|
||
| grep -m1 '"digest"' \
|
||
| awk -F '"' '{print $4}' \
|
||
|| true
|
||
}
|
||
|
||
get_local_digest() {
|
||
local image="$1"
|
||
docker inspect --format='{{index .RepoDigests 0}}' "$image" 2>/dev/null \
|
||
| cut -d'@' -f2
|
||
}
|
||
|
||
pull_with_retry() {
|
||
local image="$1"
|
||
|
||
local retries=3
|
||
local delay=3
|
||
|
||
for ((i=1; i<=retries; i++)); do
|
||
|
||
if docker pull "$image" >/dev/null 2>&1; then
|
||
return 0
|
||
fi
|
||
|
||
if [ "$i" -lt "$retries" ]; then
|
||
log WARN " ⚠️ Pull fehlgeschlagen – Versuch $i/$retries, retry in ${delay}s"
|
||
sleep "$delay"
|
||
fi
|
||
done
|
||
|
||
return 1
|
||
}
|
||
|
||
rate_limit() {
|
||
sleep 0.2
|
||
}
|
||
|
||
|
||
# =============================
|
||
# Start
|
||
# =============================
|
||
|
||
log INFO "==== Docker Compose Update gestartet ===="
|
||
|
||
script_start=$(date +%s)
|
||
|
||
notify_stacks_updated=()
|
||
notify_excluded_updates=()
|
||
error_flag=false
|
||
|
||
declare -A stack_tree
|
||
stack_tree=()
|
||
|
||
cd "$PATH_COMPOSE_DIR"
|
||
|
||
while IFS= read -r -d '' file; do
|
||
|
||
dir=$(dirname "$file")
|
||
stack=$(basename "$dir")
|
||
|
||
if is_stack_excluded "$stack"; then
|
||
log INFO "→ Stack $stack übersprungen (excluded)"
|
||
continue
|
||
fi
|
||
|
||
log INFO ""
|
||
log INFO "→ Prüfe Stack: $stack"
|
||
|
||
stack_start=$(date +%s)
|
||
|
||
cd "$dir"
|
||
|
||
compose_json=$(docker compose config --format json)
|
||
|
||
mapfile -t services < <(docker compose config --services)
|
||
total_services=${#services[@]}
|
||
current_index=0
|
||
|
||
declare -A was_running
|
||
for svc in "${services[@]}"; do
|
||
is_running "$svc" && was_running[$svc]=1 || was_running[$svc]=0
|
||
done
|
||
|
||
stack_updated=false
|
||
changed_services=()
|
||
stack_block=""
|
||
update_lines=()
|
||
|
||
for svc in "${services[@]}"; do
|
||
|
||
update_needed=false
|
||
|
||
current_index=$((current_index + 1))
|
||
|
||
if [ "$current_index" -eq "$total_services" ]; then
|
||
prefix="└─"
|
||
else
|
||
prefix="├─"
|
||
fi
|
||
|
||
image=$(get_image "$svc")
|
||
|
||
if [ -z "$image" ]; then
|
||
log WARN " $prefix $svc → kein Image"
|
||
continue
|
||
fi
|
||
|
||
if is_excluded "$svc"; then
|
||
pull_with_retry "$image" || true
|
||
notify_excluded_updates+=("$stack/$svc")
|
||
log INFO " $prefix $svc (excluded)"
|
||
continue
|
||
fi
|
||
|
||
log INFO " $prefix $svc ($image)"
|
||
|
||
before_id=$(get_container_image_id "$svc")
|
||
|
||
# 👉 Skip wenn kein Container und nicht erlaubt
|
||
if [ -z "$before_id" ] && [ "$UPDATE_INCLUDE_STOPPED" = false ]; then
|
||
log INFO " ⏭️ übersprungen (kein Container vorhanden)"
|
||
continue
|
||
fi
|
||
|
||
# =============================
|
||
# Digest Check
|
||
# =============================
|
||
|
||
local_digest=$(get_local_digest "$image")
|
||
|
||
if [ -z "$local_digest" ]; then
|
||
update_needed=true
|
||
|
||
elif [ -n "$before_id" ]; then
|
||
# 👉 nur bei laufenden Containern → teurer Check
|
||
remote_digest=$(get_remote_digest "$image" || true)
|
||
rate_limit
|
||
|
||
if [ -z "$remote_digest" ]; then
|
||
log DEBUG " ⚠️ kein Remote Digest → fallback auf pull"
|
||
update_needed=true
|
||
elif [ "$remote_digest" != "$local_digest" ]; then
|
||
update_needed=true
|
||
else
|
||
update_needed=false
|
||
fi
|
||
|
||
else
|
||
# 👉 gestoppter Container → KEIN manifest check
|
||
update_needed=true
|
||
fi
|
||
|
||
# =============================
|
||
# Pull + echte Prüfung
|
||
# =============================
|
||
|
||
if [ "$update_needed" = true ]; then
|
||
|
||
if ! pull_with_retry "$image"; then
|
||
log ERROR " ❌ Pull fehlgeschlagen"
|
||
error_flag=true
|
||
continue
|
||
fi
|
||
|
||
after_id=$(get_local_image_id "$image")
|
||
|
||
# 👉 Nur vergleichen wenn Container existiert
|
||
if [ -n "$before_id" ]; then
|
||
if [ "$before_id" != "$after_id" ]; then
|
||
update_needed=true
|
||
else
|
||
update_needed=false
|
||
fi
|
||
else
|
||
# 👉 Kein Container → kein echtes "Update"
|
||
update_needed=false
|
||
fi
|
||
fi
|
||
|
||
# =============================
|
||
# Update erkannt
|
||
# =============================
|
||
|
||
if [ "$update_needed" = true ]; then
|
||
|
||
stack_updated=true
|
||
changed_services+=("$svc")
|
||
|
||
log INFO " ⬆️ UPDATE"
|
||
log INFO " alt: ${image}@${before_id}"
|
||
log INFO " neu: ${image}@${after_id}"
|
||
|
||
short_before="${before_id#sha256:}"
|
||
short_before="${short_before:0:6}"
|
||
short_after="${after_id#sha256:}"
|
||
short_after="${short_after:0:6}"
|
||
|
||
update_lines+=("$svc|$short_before|$short_after")
|
||
fi
|
||
done
|
||
|
||
# =============================
|
||
# Update Logik
|
||
# =============================
|
||
|
||
if [ "$stack_updated" = true ]; then
|
||
|
||
if [ "$total_services" -eq 1 ]; then
|
||
svc="${services[0]}"
|
||
log INFO " 🔄 Einzelcontainer-Update: $svc"
|
||
|
||
if [ "${was_running[$svc]}" = 1 ]; then
|
||
run_cmd docker compose up -d "$svc" --remove-orphans
|
||
else
|
||
if [ "$UPDATE_START_STOPPED" = true ]; then
|
||
run_cmd docker compose up -d "$svc" --remove-orphans
|
||
else
|
||
run_cmd docker compose create "$svc"
|
||
fi
|
||
fi
|
||
|
||
log INFO " ✔️ Container $svc aktualisiert"
|
||
|
||
if [ "$REDEPLOY_WAIT_HEALTHY" = true ]; then
|
||
wait_for_healthy "$REDEPLOY_TIMEOUT" "$svc"
|
||
fi
|
||
|
||
else
|
||
log INFO " 🔄 Stack wird neu deployt (Trigger: ${changed_services[*]})"
|
||
log INFO " ⏳ Deploy läuft..."
|
||
|
||
if ! run_cmd docker compose up -d --remove-orphans >/dev/null 2>&1; then
|
||
log ERROR " ❌ Stack Update fehlgeschlagen"
|
||
error_flag=true
|
||
else
|
||
log INFO " ✔️ Stack erfolgreich aktualisiert"
|
||
|
||
if [ "$REDEPLOY_WAIT_HEALTHY" = true ]; then
|
||
wait_for_healthy "$REDEPLOY_TIMEOUT" "${changed_services[@]}"
|
||
fi
|
||
|
||
for svc in "${services[@]}"; do
|
||
if [ "${was_running[$svc]}" = 0 ]; then
|
||
log INFO " ⏹️ Stoppe $svc (war vorher gestoppt)"
|
||
run_cmd docker compose stop "$svc" >/dev/null 2>&1 || true
|
||
fi
|
||
done
|
||
fi
|
||
fi
|
||
|
||
# =============================
|
||
# Baum für ntfy bauen
|
||
# =============================
|
||
|
||
if [ ${#update_lines[@]} -gt 0 ]; then
|
||
stack_block=""
|
||
|
||
for i in "${!update_lines[@]}"; do
|
||
IFS="|" read -r svc before after <<< "${update_lines[$i]}"
|
||
|
||
if [ "$i" -eq $((${#update_lines[@]} - 1)) ]; then
|
||
prefix="└─"
|
||
else
|
||
prefix="├─"
|
||
fi
|
||
|
||
if [ "$NTFY_SHOW_VERSIONS" = true ]; then
|
||
stack_block+=$'\n'"$prefix $svc ($before → $after)"
|
||
else
|
||
stack_block+=$'\n'"$prefix $svc"
|
||
fi
|
||
done
|
||
|
||
stack_tree["$stack"]="$stack_block"
|
||
notify_stacks_updated+=("$stack")
|
||
fi
|
||
fi
|
||
|
||
cd "$PATH_COMPOSE_DIR"
|
||
|
||
stack_end=$(date +%s)
|
||
log INFO " ⏱ Dauer: $((stack_end - stack_start))s"
|
||
|
||
done < <(find . -name "$PATH_COMPOSE_PATTERN" -print0 | sort -z)
|
||
|
||
# =============================
|
||
# Cleanup
|
||
# =============================
|
||
|
||
freed_space="0"
|
||
|
||
if [ "$CLEANUP_ENABLED" = true ]; then
|
||
|
||
if [ "$CLEANUP_ONLY_ON_UPDATE" = true ] && \
|
||
[ ${#notify_stacks_updated[@]} -eq 0 ]; then
|
||
log INFO "🧹 Cleanup übersprungen (keine Updates)"
|
||
else
|
||
|
||
before_size=$(get_docker_disk_usage)
|
||
log INFO "🧹 Docker Cleanup läuft..."
|
||
|
||
if [ "$CLEANUP_IMAGES_ENABLED" = true ]; then
|
||
case "$CLEANUP_IMAGES_MODE" in
|
||
unused) run_cmd docker image prune -a -f >/dev/null 2>&1 ;;
|
||
dangling) run_cmd docker image prune -f >/dev/null 2>&1 ;;
|
||
esac
|
||
fi
|
||
|
||
if [ "$CLEANUP_CONTAINERS_ENABLED" = true ]; then
|
||
run_cmd docker container prune -f >/dev/null 2>&1
|
||
fi
|
||
|
||
if [ "$CLEANUP_VOLUMES_ENABLED" = true ]; then
|
||
run_cmd docker volume prune -f >/dev/null 2>&1
|
||
fi
|
||
|
||
if [ "$CLEANUP_NETWORKS_ENABLED" = true ]; then
|
||
run_cmd docker network prune -f >/dev/null 2>&1
|
||
fi
|
||
|
||
after_size=$(get_docker_disk_usage)
|
||
freed_space=$((after_size < before_size ? before_size - after_size : 0))
|
||
|
||
log INFO "✔️ Cleanup abgeschlossen (${freed_space} MB freigegeben)"
|
||
fi
|
||
fi
|
||
|
||
# =============================
|
||
# Notification
|
||
# =============================
|
||
|
||
if [ "$NTFY_ENABLED" = true ]; then
|
||
|
||
msg=""
|
||
|
||
if [ ${#stack_tree[@]} -gt 0 ]; then
|
||
msg+=$'\n🔄 Stack Updates\n'
|
||
|
||
for stack in $(printf "%s\n" "${!stack_tree[@]}" | sort); do
|
||
msg+=$'\n'"$stack"
|
||
msg+="${stack_tree[$stack]}"
|
||
msg+=$'\n' # Leerzeile nach jedem Stack
|
||
done
|
||
fi
|
||
|
||
if [ ${#notify_excluded_updates[@]} -gt 0 ]; then
|
||
msg+=$'\n\n⏭️ Excluded (Update verfügbar)'
|
||
for s in "${notify_excluded_updates[@]}"; do
|
||
msg+=$'\n - '"$s"
|
||
done
|
||
fi
|
||
|
||
if [ ${#stack_tree[@]} -eq 0 ] && \
|
||
[ ${#notify_excluded_updates[@]} -eq 0 ]; then
|
||
msg+=$'\n\n✔️ Alles aktuell'
|
||
fi
|
||
|
||
if [ "$freed_space" != "0" ]; then
|
||
msg+=$'\n\n🧹 Cleanup: '"${freed_space} MB freigegeben"
|
||
fi
|
||
|
||
if [ "$error_flag" = true ]; then
|
||
PRIORITY=5
|
||
elif [ ${#stack_tree[@]} -gt 0 ]; then
|
||
PRIORITY=3
|
||
else
|
||
PRIORITY=1
|
||
fi
|
||
|
||
send_ntfy "$msg" "$PRIORITY"
|
||
log INFO "ntfy Nachricht gesendet (prio=$PRIORITY)"
|
||
fi
|
||
|
||
script_end=$(date +%s)
|
||
log INFO "⏱ Gesamtzeit: $((script_end - script_start))s"
|
||
log INFO "==== Update beendet ====" |