source: trunk/SH/arb_launcher

#!/bin/bash

set -u

trace() {
    local MSG=$1
    echo "[arb_launcher[${ARB_LAUNCHER:-}]: $1]"
}
debug() {
    local MSG=$1
    # to debug uncomment next line:
    # trace "DEBUG: $MSG"
}

send_to_launcher() {
    local NAMED_PIPE=$1
    local CMD=$2

    debug "Sending '$CMD' to $NAMED_PIPE"
    echo "$CMD" >$NAMED_PIPE
    sleep 1
    debug "send_to_launcher terminates"
}

pipe_command() {
    local NAMED_PIPE=$1; shift
    local CMD=$1; shift
    local LOGDIR=${1:-}; shift # LOGDIR may be empty/undef -> dont signal crash

    trace "Starting '$CMD'.."
    $CMD
    local EXITCODE=${PIPESTATUS[0]}
    if [ $EXITCODE == 0 ]; then
        trace "'$CMD' has terminated with success"
    else
        trace "'$CMD' has terminated with error $EXITCODE"
        if [ -n "$LOGDIR" ]; then
            if [ $EXITCODE = 1 ]; then
                touch $LOGDIR/failed
            else
                touch $LOGDIR/crashed
            fi
        fi
    fi

    send_to_launcher $NAMED_PIPE 'cmd_terminated'
    debug "pipe_command terminates"
}

read_line() {
    local NAMED_PIPE=$1
    local LINE=""

    if read ATTEMPT <$NAMED_PIPE; then
        LINE=$ATTEMPT
    fi
    echo $LINE
}

listen_pipe_unlogged() {
    local NAMED_PIPE=$1; shift
    local LOGDIR=${1:-}; shift # LOGDIR may be empty/undef -> dont log 
    local RUNNING=1
    local STARTED=0
    # RUNNING is set to 1 (otherwise listen_pipe would terminate instantly)

    trace "log for ARB_PID='${ARB_PID}'"

    while (($RUNNING > 0))
      do
      LINE=`read_line $NAMED_PIPE 2>/dev/null`
      if [[ ! -z "$LINE" ]]; then
          debug "'$NAMED_PIPE' received '$LINE'"
          if [[ "$LINE" == 'TERMINATE' ]]; then
              trace "Received request to TERMINATE"
              break;
          else
              if [[ "$LINE" == 'cmd_terminated' ]]; then
                  RUNNING=$(($RUNNING - 1))
                  if (($RUNNING>0)); then
                      trace "Still have $RUNNING arb processes.."
                  fi
              else
                  if [[ "$LINE" == 'allow_termination' ]]; then
                      RUNNING=$(($RUNNING - 1))
                  else
                      pipe_command $NAMED_PIPE "$LINE" $LOGDIR &
                      RUNNING=$(($RUNNING + 1))
                      STARTED=$(($STARTED + 1))
                      debug "RUNNING=$RUNNING"
                      debug "STARTED=$STARTED"
                  fi
              fi
          fi
      fi
    done

    if (($RUNNING==0)); then
        if (($STARTED>0)); then
            trace "All launched processes terminated"
        else
            trace "Nothing was ever launched"
        fi
    else
        trace "Still have $RUNNING arb-processes - terminating nevertheless"
    fi

    debug "listen_pipe_unlogged waits for subshells ..."
    wait

    trace "cleaning up arb session"
    arb_clean show_session
    arb_clean session

    debug "listen_pipe_unlogged terminates"
}

if_command() {
    local CMD=$1; shift

    local FULL_CMD=$($ARBHOME/bin/arb_path.sh -x ${CMD})
    if [ -n "${FULL_CMD}" ]; then
        local CALL="${FULL_CMD} $@"
        echo "{${CALL}}"
        ${CALL}
    # else
        # echo "{no ${CMD}}"
    fi
    # never report errors from here
    true
}

desktop_info() {
    echo "XDG_CURRENT_DESKTOP='${XDG_CURRENT_DESKTOP:-}'"
    echo "GDMSESSION='${GDMSESSION:-}'"
    # for combinations occurring on different systems see
    # https://askubuntu.com/questions/72549/how-to-determine-which-window-manager-is-running/227669#227669

    if_command konqueror --version
    if_command gnome-panel --version
    if_command xfce4-about --version
    if_command cinnamon --version
}

shared_library_dependencies() {
    case `uname` in
        Linux)
            LIST_DYNLIBS="ldd"
            BINARIES="bin/arb_ntree lib/libARBDB.so lib/libCORE.so lib/libWINDOW.so"
            ;;
        Darwin)
            LIST_DYNLIBS="otool -L"
            # Darwin ARB links internal stuff static
            BINARIES="bin/arb_ntree"
            ;;
        *)
            LIST_DYNLIBS="echo UNSUPPORTED_OS "
            ;;
    esac
    for binary in $BINARIES; do
        echo -e "Library dependencies for $ARBHOME/$binary:"
        $LIST_DYNLIBS $ARBHOME/$binary
    done
}

wrapped_info() {
    local TAG=$1; shift
    local CMD=$1; shift
    echo "--------------------"
    echo "[$TAG start]"
    eval $CMD
    echo "[$TAG end]"
    echo ""
}

collect_system_information() {
    echo "System information"
    echo ""
    echo "The information below has been collected by ARB."
    echo "Please do not publish without being aware that it might contain personal information."
    echo "[current date: `date`]"
    echo ""

    local ARB_RELEVANT="| grep -i ARB"

    wrapped_info "version" "$ARBHOME/bin/arb_ntree --help"
    wrapped_info "environment" "printenv $ARB_RELEVANT"
    wrapped_info "sina" "$ARBHOME/bin/arb_sina.sh runldd"
    wrapped_info "OS" "lsb_release -a"
    wrapped_info "kernel" "uname -mrs ; uname -a ; cat /proc/version"
    wrapped_info "desktop" "desktop_info"

    wrapped_info "shared libraries" "shared_library_dependencies"
    wrapped_info "disk" "df -h"
    wrapped_info "memory" "free -m ; cat /proc/meminfo"
    wrapped_info "user limits" "ulimit -a"
    wrapped_info "ARB processes" "ps aux $ARB_RELEVANT"

    wrapped_info "CPU" "cat /proc/cpuinfo"
    wrapped_info "X server" "xdpyinfo"
    # wrapped_info "X" "Y"
}

erase_old_logs() {
    local LOGBASE=$1
    if [ -d "$LOGBASE" ]; then
        # remove files older than 15 days inside and below LOGBASE
        local OLD=$(( 60 * 24 * 15 ))
        find $LOGBASE -type f -cmin +$OLD -exec rm {} \;
        # remove empty directories inside and below LOGBASE
        find $LOGBASE -type d -depth -empty -mindepth 1 -exec rmdir {} \;
    fi
}

listen_pipe() {
    # this is just a wrapper around listen_pipe_unlogged.
    # wrapper performs ARB session logging
    local NAMED_PIPE=$1

    if [ -z ${ARB_PROP:-} ]; then
        # should never come here, if arb has been started via script 'arb'
        # (e.g. happens when arb_ntree was started from debugger and then 'start second database' has been called)
        listen_pipe_unlogged $NAMED_PIPE
    else
        local LOGBASE=$ARB_PROP/logs
        local LOGDIRID=`date '+%Y%m%d_%H%M%S'`.$$
        local LOGDIR=$LOGBASE/$LOGDIRID
        local NTREE_STATUS=

        mkdir -p $LOGDIR

        if [ -d "$LOGDIR" ]; then
            local RUNLOG=$LOGDIR/run.log
            local SERVERLOG=$LOGDIR/server.log
            local SYSLOG=$LOGDIR/sys.info
            local CRASHFLAG=$LOGDIR/crashed
            local FAILFLAG=$LOGDIR/failed

            # tell arb to start servers as logging daemons
            export ARB_SERVER_LOG=$SERVERLOG
            echo "`date` arb server.log created by arb_launcher" > $SERVERLOG

            # forward server output to launcher-tty (non-blocking)
            tail -f $SERVERLOG &
            local TAILPID=$!

            ( ( collect_system_information 2>&1 ) > $SYSLOG ; erase_old_logs $LOGBASE ) &
            ( listen_pipe_unlogged $NAMED_PIPE $LOGDIR ) 2>&1 | tee $RUNLOG

            if [ -e $CRASHFLAG ]; then
                # only detects crashes of arb_ntree
                # (clients are not started via arb_launcher and they usually crash when server exits)
                NTREE_STATUS=crash
            else
                if [ -e $FAILFLAG ]; then
                    NTREE_STATUS=fail
                fi
            fi

            if [ "$NTREE_STATUS" != "" ]; then
                echo "abnormal termination (NTREE_STATUS='$NTREE_STATUS')" >> $RUNLOG
            else
                echo "normal termination" >> $RUNLOG
            fi

            local TARBALLNAME=session.$LOGDIRID.tgz

            debug "killing tail on server-log (pid=$TAILPID)"
            kill ${TAILPID}

            echo "`date` arb_launcher terminates now. leftover servers may continue logging into this file" >> $SERVERLOG
            echo "`date` End of log (now archive into $LOGBASE/$TARBALLNAME)" >> $RUNLOG

            ( cd $LOGBASE ; tar -zcf $TARBALLNAME $LOGDIRID )
            rm -f $RUNLOG $SYSLOG $CRASHFLAG $FAILFLAG
            rmdir $LOGDIR

            local FULLTARBALL=$LOGBASE/$TARBALLNAME
            echo ""
            echo "Session log has been stored in $FULLTARBALL"

            local LATESTLINK=~/ARB_last_session.tgz
            if [ -h $LATESTLINK ]; then
                rm $LATESTLINK
            fi
            if [ -e $LATESTLINK ]; then
                echo "$LATESTLINK already exists and is no symlink"
            else
                (cd ~; ln -s $FULLTARBALL $LATESTLINK )
                echo "    and is also accessible via $LATESTLINK"
            fi

            if [ "$NTREE_STATUS" != "" ]; then
                echo ""
                if [ $NTREE_STATUS = "crash" ]; then
                    echo "ARB crashed :-("
                    echo "To report this goto http://bugs.arb-home.de/wiki/BugReport"
                    echo "Please include the session log(s) mentioned above!"
                    echo ""
                else
                    echo "ARB terminated abnormally"
                fi

                echo "[$(date)]"
                local DAYWAIT=3
                echo "[press ENTER or wait ${DAYWAIT} days]"
                local SECWAIT=$((${DAYWAIT}*24*60*60))
                read -t ${SECWAIT} A
            fi

            true
        else
            echo "Error creating directory '$LOGDIR'"
            false
        fi
    fi
}

killtree() {
    local _pid=$1
    local _sig=${2:-TERM}

    debug "killtree pid=${_pid} with sig=${_sig} pid=$$"
    kill -stop ${_pid} # stop quickly forking parent from producing childs
    killchilds ${_pid} ${_sig}
    kill ${_sig} ${_pid}
}
killchilds() {
    local _pid=$1
    local _sig=${2:-TERM}

    debug "killchilds pid=${_pid} with sig=${_sig} pid=$$"
    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
        killtree ${_child} ${_sig}
    done
}

term_handler() {
    local NAMED_PIPE=$1

    trace "Killing ARB session for ARB_PID=$ARB_PID"
    arb_clean session
    debug "arb_clean done - now killing process tree"
    killchilds $$ -TERM
    debug "killchilds done - exiting $$"
    exit
}

create_pipe_reader() {
    local NAMED_PIPE=$1
    local PARENT_PID=$2

    if [ -z "${ARB_LAUNCHER:-}" ]; then
        export ARB_LAUNCHER=0
    else
        export ARB_LAUNCHER=$(($ARB_LAUNCHER+1))
    fi

    debug "Creating named pipe '$NAMED_PIPE'"

    # (i did not manage to recover from SIGINT w/o termination of listen_pipe)
    # => disable SIGINT handler
    trap '' INT
    trap "term_handler $NAMED_PIPE" TERM
    trap "rm -f $NAMED_PIPE" EXIT

    { mkfifo -m 600 $NAMED_PIPE && listen_pipe $NAMED_PIPE ; debug "listen_pipe done" ; } || \
      { echo "Error creating pipe '$NAMED_PIPE'" ; kill $PARENT_PID ; }

    debug "Pipe reader for '$NAMED_PIPE' terminates.."
    rm -f $NAMED_PIPE
    debug "Pipe '$NAMED_PIPE' removed"
}

initial_send_to_launcher() {
    local NAMED_PIPE=$1
    local CMD=$2

    send_to_launcher $NAMED_PIPE "$CMD"

    # now allow pipe reader to terminate:
    send_to_launcher $NAMED_PIPE "allow_termination"
}

wait_for_pipe() {
    local NAMED_PIPE=$1

    while [[ ! -p $NAMED_PIPE ]];
      do
      echo "Waiting for '$NAMED_PIPE'.."
      sleep 1
    done
    debug "pipe is open"
}

get_pipe_name() {
    local SOCKETDIR="$HOME/.arb_tmp/sockets"
    mkdir -p "$SOCKETDIR"
    chmod 0700 "$SOCKETDIR"
    echo "$SOCKETDIR/arb_launcher.$ARB_PID"

    # instead of the above code, use the following to test a pipe-creation failure:
    # echo "/arb_launcher.$ARB_PID"
}

launcher() {
    local ASYNC=0
    if [ "$1" = "--async" ]; then
        ASYNC=1
        shift
    fi
    local CMD="$*"

    if [ -z "$ARB_PID" ]; then
        echo "Error: environment variable ARB_PID is unset. terminating.."
        false
    else
        if [ -z "$1" ]; then
            echo "Usage: arb_launcher \"shellcommand\""
            echo ""
            echo "          runs 'shellcommand'"
            echo "          "
            echo "          The initial call to arb_launcher will block until 'shellcommand' terminates."
            echo ""
            echo "          Subsequent calls will not block. They are started from the context of the"
            echo "          initial call. The initial call will wait for all started commands."
            echo ""
            echo "       arb_launcher \"TERMINATE\""
            echo ""
            echo "          terminate the launcher without waiting for spawned commands."
            echo ""
        else
            debug "Using ARB_PID '$ARB_PID'"
            local NAMED_PIPE=$(get_pipe_name)
            debug "Using NAMED_PIPE '$NAMED_PIPE'"

            if [[ ! -p $NAMED_PIPE ]]; then
                ( wait_for_pipe $NAMED_PIPE ; initial_send_to_launcher $NAMED_PIPE "$CMD" ) &
                if (( $ASYNC==1 )); then
                    create_pipe_reader $NAMED_PIPE $$ &
                else
                    create_pipe_reader $NAMED_PIPE $$
                fi
            else
                debug "pipe already was open"
                send_to_launcher $NAMED_PIPE "$CMD"
            fi

            # if pipe-reader was started from current process
            # -> blocks until all launched processes have terminated
            if (( $ASYNC==0 )); then
                wait
            fi
        fi
    fi
}

launcher "$@"
debug "arb_launcher exits!"