Public release

apply

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function usage() {
+    echo "usage: $(basename "$0") [-v] [-p] <host> [<host>...]"
+    exit 1
+}
+
+function host_targets() {
+    local host="$*"
+
+    cat < "$host" | tr '\n' ' '
+}
+
+function push_each() {
+    local vflag="$1"
+    local parallelize="$2"
+    shift
+    shift
+    if [[ -n $parallelize ]]; then
+        # shellcheck disable=SC2086
+        parallel -j10 --progress --colsep ' ' ./push $vflag
+    else
+        # shellcheck disable=SC2086
+        parallel -j1 -u --colsep ' ' ./push $vflag
+    fi
+}
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+if [[ "$1" == "-v" ]]; then
+    vflag='-v'
+    shift
+else
+    vflag=''
+fi
+
+if [[ "$1" == "-p" ]]; then
+    pflag='-p'
+    shift
+else
+    pflag=''
+fi
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+hosts=()
+while [[ $# -gt 0 ]]; do
+    hosts+=("$1")
+    shift
+done
+
+for host in ${hosts[*]}; do
+    target=${host#*/}
+    # shellcheck disable=SC2046
+    echo $(host_targets "$host") root@"$target"
+done | push_each "$vflag" "$pflag"

lib

@@ -0,0 +1,46 @@
+# shellcheck shell=bash
+# vim: ft=sh
+
+# Support functions
+
+function ssh_version() {
+    ssh -V 2>&1 | perl -ne '/^OpenSSH_(\d+\.\d+)/ and print "$1";'
+}
+
+function ssh_back() {
+    local remote="$1"
+    local timeout=60
+    local port=0
+    shift
+
+    case $(ssh_version) in
+        5.*)
+            port=55555
+			# shellcheck disable=SC2029
+            ssh -f -R "$port":127.0.0.1:22 "$remote" sleep "$timeout" >/dev/null 2>&1
+            ;;
+        *)
+			# shellcheck disable=SC2029
+            port="$(ssh -f -R 0:127.0.0.1:22 "$remote" sleep "$timeout" 2>&1 >/dev/null | head -1 | perl -ne '/Allocated port (\d+)/ and print "$1"')"
+            ;;
+    esac
+
+    local args="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p $port"
+	# shellcheck disable=SC2029
+    [[ -n "$port" ]] && ssh -A "$remote" "SSH_BACK_PORT=$port; SSH_BACK_USER=$USER; SSH_BACK_ARGS='$args'; $*" 2> >(grep -v "Permanently added")
+}
+
+# Fallback functions
+
+if ! which systemctl >/dev/null; then
+function systemctl() {
+    case "$1" in
+        start|stop|restart|reload)
+            service "$2" "$1"
+            ;;
+        *)
+            false
+            ;;
+    esac
+}
+fi

push

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function usage() {
+    echo "usage: $(basename "$0") [-v] <target> [<target>...] <user@remote>"
+    exit 1
+}
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+if [[ "$1" == "-v" ]]; then
+    vflag='-v'
+    shift
+else
+    vflag=''
+fi
+
+if [[ $# -lt 2 ]]; then
+    usage
+fi
+
+targets=()
+while [[ $# -gt 1 ]]; do
+    targets+=("$1")
+    shift
+done
+
+remote="$1"
+
+tmp=$(ssh "$remote" 'mktemp -d')
+echo -e -n "\033[33m** pushing to\033[0m $remote:$tmp"
+if scp -q -r groups units run lib "$remote":"$tmp"; then
+    echo -e " \033[32mOK\033[0m"
+fi
+
+# shellcheck disable=SC2029
+ssh -A "$remote" "cd '$tmp'; ./run $vflag ${targets[*]}"

run

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+set -e
+set -u
+
+function usage() {
+    echo "usage: $(basename "$0") [-v] <target> [<target>...]"
+    exit 1
+}
+
+function root_path() {
+    cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
+}
+
+function read_group() {
+    local target="$*"
+
+    while read -r item; do
+        if [[ ! "$item" == '#'* ]]; then
+            echo "$item"
+        fi
+    done < "$(root_path)/$target" 
+}
+
+function run_group() {
+    local target="$*"
+
+    for item in $(read_group "$target"); do
+        if [[ ! "$item" == '#'* ]]; then
+            run_pretty "$item"
+        fi
+    done
+}
+
+function run_unit() {
+    /bin/bash -l -c "set -e; set -u; set -o pipefail; source lib; source '$*'"
+}
+
+function log_file() {
+    echo -n "$LOG_DIR/"
+    echo "${1/\//_}.log"
+}
+
+function check_target() {
+    local target="$*"
+
+    if [[ ! -f "$(root_path)/$target" ]]; then
+        echo "missing $target"
+        return 1
+    fi
+
+    if [[ $target == groups/* ]]; then
+        for item in $(read_group "$target"); do
+            check_target "$item"
+        done
+    fi
+}
+
+function run_pretty() {
+    local target="$*"
+
+    case "$target" in
+        groups/*)
+            echo -e "\033[33m** \033[34m$target \033[33mprocessing...\033[0m"
+            run_group "$target"
+            echo -e "\033[33m** \033[34m$target \033[32mOK\033[0m"
+            ;;
+
+        units/*)
+            run_pretty_unit "$target"
+            ;;
+        *)
+            echo "unsupported command: $target"
+            ;;
+    esac
+}
+
+function run_pretty_unit() {
+    local rc
+    local log_file
+    local target="$*"
+
+    echo -e -n "\033[33m** \033[34m$target\033[0m"
+    [[ "$VERBOSE" == "1" ]] && echo -e ": \033[33mstarting...\033[0m"
+
+    log_file=$(log_file "$@")
+
+    if [[ "$VERBOSE" == "1" ]]; then
+        set +e
+        run_unit "$@"
+        rc=$?
+        set -e
+    else
+        set +e
+        run_unit "$@" >"$log_file" 2>&1
+        rc=$?
+        set -e
+    fi
+
+    if [[ "$VERBOSE" == "1" ]]; then
+        echo -e -n "\033[33m** \033[34m$target\033[0m: "
+    else
+        echo -n " "
+    fi
+
+    if [[ $rc -eq 0 ]]; then
+        echo -e "\033[32mOK\033[0m"
+    else
+        echo -e "\033[31mFAILED\033[0m"
+        if [[ "$VERBOSE" == "1" ]]; then
+            echo "For details, see output above" 1>&2
+        else
+            echo "Here are the last lines of output:" 1>&2
+            tail -n20 "$log_file" 1>&2
+            echo "For details, see $log_file" 1>&2
+        fi
+        exit $rc
+    fi
+}
+
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+if [[ "$1" == "-v" ]]; then
+    VERBOSE="1"
+    shift
+else
+    VERBOSE="0"
+fi
+
+if [[ $# -lt 1 ]]; then
+    usage
+fi
+
+targets=( "$@" )
+
+# pre-flight checks
+for target in "${targets[@]}"; do
+    check_target "$target"
+done
+
+# fly!
+LOG_DIR=$(mktemp -d)
+for target in "${targets[@]}"; do
+    run_pretty "$target"
+done