From 790782fada525a9b0a0671015d411c2948c27b8b Mon Sep 17 00:00:00 2001 From: cortex Date: Wed, 20 May 2026 21:49:06 +0100 Subject: [PATCH] Worker-gen rewrite --- init/scripts/config_gen.sh | 4 +- init/scripts/worker_gen.sh | 100 ++++++++++++++++++++++++------------- 2 files changed, 67 insertions(+), 37 deletions(-) diff --git a/init/scripts/config_gen.sh b/init/scripts/config_gen.sh index da2d1e2..9949a2e 100755 --- a/init/scripts/config_gen.sh +++ b/init/scripts/config_gen.sh @@ -51,7 +51,7 @@ docker run -it --rm --network matrix_network \ -v "${VOLUME_PATH}/synapse:/data" \ -e SYNAPSE_SERVER_NAME="$HOMESERVER_FQDN" \ -e SYNAPSE_REPORT_STATS=no \ - ghcr.io/element-hq/synapse:latest generate + ghcr.io/element-hq/synapse:latest generate || { log_error "Synapse config generation failed"; exit 1; } fix_ownership "${VOLUME_PATH}/synapse" @@ -59,7 +59,7 @@ log_info "Generating MAS configuration..." docker run -it --rm --network matrix_network \ -v "${VOLUME_PATH}/mas:/data" \ ghcr.io/element-hq/matrix-authentication-service:latest config generate \ - | grep -v '^[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}' > "${VOLUME_PATH}/mas/config.yaml" + | grep -v '^[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}' > "${VOLUME_PATH}/mas/config.yaml" || { log_error "Mas config generation failed"; exit 1; } fix_ownership "${VOLUME_PATH}/mas" log_info "Configuration complete" diff --git a/init/scripts/worker_gen.sh b/init/scripts/worker_gen.sh index b9460e5..c16d8ae 100755 --- a/init/scripts/worker_gen.sh +++ b/init/scripts/worker_gen.sh @@ -4,16 +4,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/functions.sh" resolve_paths - -WORKER_COUNT=1 -while getopts "c:" opt; do - case $opt in - c) WORKER_COUNT="$OPTARG" ;; - *) echo "Usage: $0 [-c count]"; exit 1 ;; - esac -done - require_env +require_command "yq" resolve_volume_path TEMPLATE_COMPOSE="${INIT_DIR}/templates/worker.yaml" @@ -25,24 +17,29 @@ require_file "$TEMPLATE_CONFIG" "Config template" require_file "$TEMPLATE_LOG" "Log template" WORKERS_DIR="${VOLUME_PATH}/synapse/workers" -mkdir -p "$WORKERS_DIR" "${ROOT_DIR}/.tmp" +mkdir -p "$WORKERS_DIR" "${SCRIPT_DIR}/tmp" -TRAEFIK_ENABLED="${PROXY:-0}" -TEMP_COMPOSE="${ROOT_DIR}/.tmp/compose_workers.yaml" +TEMP_COMPOSE="${SCRIPT_DIR}/tmp/compose_workers.yaml" +# Initialize empty compose file for workers > "$TEMP_COMPOSE" for ((i=1; i<=WORKER_COUNT; i++)); do worker_name="synapse-worker-${i}" port=$((9093 + i)) + TEMP_WORKER_COMPOSE="${SCRIPT_DIR}/tmp/${worker_name}.yaml" + log_info "Generating configuration for ${worker_name}..." + + # 1. Generate Worker Config (synapse config file) sed -e "s/{{WORKER_NAME}}/${worker_name}/g" \ -e "s/{{PORT}}/${port}/g" \ -e "s/{{SERVER_NAME}}/${HOMESERVER_FQDN}/g" \ - -e "s/{{DB_USER}}/${PG_USER_MATRIX}/g" - -e "s/{{DB_PASSWORD}}/${PG_PASSWORD_MATRIX}/g" - -e "s/{{DB_NAME}}/${PG_DB_MATRIX}/g" + -e "s/{{DB_USER}}/${PG_USER_MATRIX}/g" \ + -e "s/{{DB_PASSWORD}}/${PG_PASSWORD_MATRIX}/g" \ + -e "s/{{DB_NAME}}/${PG_DB_MATRIX}/g" \ "$TEMPLATE_CONFIG" > "${WORKERS_DIR}/${worker_name}.yaml" + # 2. Generate Log Config sed -e "s/{{WORKER_NAME}}/${worker_name}/g" \ -e "s/{{MAX_BYTES}}/${LOG_MAX_BYTES:-10485760}/g" \ -e "s/{{BACKUP_COUNT}}/${LOG_BACKUP_COUNT:-5}/g" \ @@ -50,37 +47,70 @@ for ((i=1; i<=WORKER_COUNT; i++)); do -e "s/{{SYNAPSE_LOG_LEVEL}}/${SYNAPSE_LOG_LEVEL:-INFO}/g" \ "$TEMPLATE_LOG" > "${WORKERS_DIR}/${worker_name}.log.config" - TRAEFIK_LABELS="" - if [[ "$PROXY" == "TRAEFIK" && -n "$DOMAIN" ]]; then - TRAEFIK_LABELS=" labels: - - traefik.enable=true - - traefik.http.services.${worker_name}.loadbalancer.server.port=${port} - - traefik.http.routers.synapse-worker-${worker_name}.rule=Host(\`${DOMAIN}\`) && PathPrefix(\`/_matrix\`) - - traefik.http.routers.synapse-worker-${worker_name}.entrypoints=websecure - - traefik.http.routers.synapse-worker-${worker_name}.service=${worker_name} - - traefik.http.routers.synapse-worker-${worker_name}.priority=10 -" - fi - + # 3. Generate Temporary Compose Snippet for this worker + # Start with the base template sed -e "s/{{WORKER_NAME}}/${worker_name}/g" \ -e "s/{{PORT}}/${port}/g" \ -e "s/{{APP_TYPE}}/synapse.app.generic_worker/g" \ - -e "s/{{TRAEFIK_LABELS}}/${TRAEFIK_LABELS}/g" \ - "$TEMPLATE_COMPOSE" >> "$TEMP_COMPOSE" + "$TEMPLATE_COMPOSE" > "$TEMP_WORKER_COMPOSE" - echo "" >> "$TEMP_COMPOSE" + # 4. Conditionally Inject Traefik Labels using yq + if [[ "$PROXY" == "TRAEFIK" && -n "$DOMAIN" ]]; then + log_info " -> Adding Traefik labels for ${worker_name}" + + # Construct the labels array safely + # Note: We escape backticks for the shell, but yq handles the YAML structure + yq -i '.services."'"${worker_name}"'".labels = [ + "traefik.enable=true", + "traefik.http.services.'"${worker_name}"'.loadbalancer.server.port='"${port}"'", + "traefik.http.routers.synapse-worker-'"${worker_name}"'.rule=Host(`'"${DOMAIN}"'`) && PathPrefix(`/_matrix`)", + "traefik.http.routers.synapse-worker-'"${worker_name}"'.entrypoints=websecure", + "traefik.http.routers.synapse-worker-'"${worker_name}"'.service='"${worker_name}"'", + "traefik.http.routers.synapse-worker-'"${worker_name}"'.priority=10" + ]' "$TEMP_WORKER_COMPOSE" + else + # Ensure labels key doesn't exist if not using Traefik + yq -i 'del(.services."'"${worker_name}"'".labels)' "$TEMP_WORKER_COMPOSE" 2>/dev/null || true + fi + + # 5. Merge this worker into the temporary aggregate compose file + # If TEMP_COMPOSE is empty, just copy. Otherwise, merge. + if [[ ! -s "$TEMP_COMPOSE" ]]; then + cp "$TEMP_WORKER_COMPOSE" "$TEMP_COMPOSE" + else + # Merge: existing content * new worker content + yq eval-all "select(fileIndex == 0) * select(fileIndex == 1)" \ + "$TEMP_COMPOSE" "$TEMP_WORKER_COMPOSE" > "${TEMP_COMPOSE}.merged" + mv "${TEMP_COMPOSE}.merged" "$TEMP_COMPOSE" + fi + + # Cleanup temp worker file + rm -f "$TEMP_WORKER_COMPOSE" done +# 6. Final Merge into main compose.yaml if [[ -f "$COMPOSE_FILE" ]]; then + # Check if workers already exist to avoid duplicates if ! grep -qE "^ synapse-worker-[0-9]+:" "$COMPOSE_FILE"; then - cat "$TEMP_COMPOSE" >> "$COMPOSE_FILE" - log_info "Appended ${WORKER_COUNT} worker(s) to compose.yaml" + # Merge main compose with generated workers + yq eval-all "select(fileIndex == 0) * select(fileIndex == 1)" \ + "$COMPOSE_FILE" "$TEMP_COMPOSE" > "${COMPOSE_FILE}.new" + + if mv "${COMPOSE_FILE}.new" "$COMPOSE_FILE"; then + log_info "Appended ${WORKER_COUNT} worker(s) to compose.yaml" + else + log_error "Failed to update compose.yaml" + exit 1 + fi else - log_warn "Some workers already exist. Manual merge required" - cp "$TEMP_COMPOSE" "${ROOT_DIR}/.tmp/compose_workers.yaml" + log_warn "Some workers already exist in compose.yaml. Manual merge required." + cp "$TEMP_COMPOSE" "${SCRIPT_DIR}/tmp/compose_workers.yaml" + log_warn "Saved new worker definitions to ${SCRIPT_DIR}/tmp/compose_workers.yaml" fi else - log_warn "compose.yaml not found. Generated: ${TEMP_COMPOSE}" + # If no main compose exists, the generated file becomes the main one + mv "$TEMP_COMPOSE" "$COMPOSE_FILE" + log_info "Created new compose.yaml with ${WORKER_COUNT} worker(s)" fi rm -f "$TEMP_COMPOSE"