source: trunk/SH/arb_launcher

Last change on this file was 19515, checked in by westram, 3 weeks ago
  • allow to reduce the time-to-wait on error, by defining ARB_SCRIPTED_SECS_WAIT_ON_ERROR.
  • Property svn:executable set to *
File size: 13.5 KB
Line 
1#!/bin/bash
2
3set -u
4
5trace() {
6    local MSG=$1
7    echo "[arb_launcher[${ARB_LAUNCHER:-}]: $1]"
8}
9debug() {
10    local MSG=$1
11    # to debug uncomment next line:
12    # trace "DEBUG: $MSG"
13}
14
15send_to_launcher() {
16    local NAMED_PIPE=$1
17    local CMD=$2
18
19    debug "Sending '$CMD' to $NAMED_PIPE"
20    echo "$CMD" >$NAMED_PIPE
21    sleep 1
22    debug "send_to_launcher terminates"
23}
24
25pipe_command() {
26    local NAMED_PIPE=$1; shift
27    local CMD=$1; shift
28    local LOGDIR=${1:-}; shift # LOGDIR may be empty/undef -> dont signal crash
29
30    trace "Starting '$CMD'.."
31    $CMD
32    local EXITCODE=${PIPESTATUS[0]}
33    if [ $EXITCODE == 0 ]; then
34        trace "'$CMD' has terminated with success"
35    else
36        trace "'$CMD' has terminated with error $EXITCODE"
37        if [ -n "$LOGDIR" ]; then
38            if [ $EXITCODE = 1 ]; then
39                touch $LOGDIR/failed
40            else
41                touch $LOGDIR/crashed
42            fi
43        fi
44    fi
45
46    send_to_launcher $NAMED_PIPE 'cmd_terminated'
47    debug "pipe_command terminates"
48}
49
50read_line() {
51    local NAMED_PIPE=$1
52    local LINE=""
53
54    if read ATTEMPT <$NAMED_PIPE; then
55        LINE=$ATTEMPT
56    fi
57    echo $LINE
58}
59
60listen_pipe_unlogged() {
61    local NAMED_PIPE=$1; shift
62    local LOGDIR=${1:-}; shift # LOGDIR may be empty/undef -> dont log
63    local RUNNING=1
64    local STARTED=0
65    # RUNNING is set to 1 (otherwise listen_pipe would terminate instantly)
66
67    trace "log for ARB_PID='${ARB_PID}'"
68
69    while (($RUNNING > 0))
70      do
71      LINE=`read_line $NAMED_PIPE 2>/dev/null`
72      if [[ ! -z "$LINE" ]]; then
73          debug "'$NAMED_PIPE' received '$LINE'"
74          if [[ "$LINE" == 'TERMINATE' ]]; then
75              trace "Received request to TERMINATE"
76              break;
77          else
78              if [[ "$LINE" == 'cmd_terminated' ]]; then
79                  RUNNING=$(($RUNNING - 1))
80                  if (($RUNNING>0)); then
81                      trace "Still have $RUNNING arb processes.."
82                  fi
83              else
84                  if [[ "$LINE" == 'allow_termination' ]]; then
85                      RUNNING=$(($RUNNING - 1))
86                  else
87                      pipe_command $NAMED_PIPE "$LINE" $LOGDIR &
88                      RUNNING=$(($RUNNING + 1))
89                      STARTED=$(($STARTED + 1))
90                      debug "RUNNING=$RUNNING"
91                      debug "STARTED=$STARTED"
92                  fi
93              fi
94          fi
95      fi
96    done
97
98    if (($RUNNING==0)); then
99        if (($STARTED>0)); then
100            trace "All launched processes terminated"
101        else
102            trace "Nothing was ever launched"
103        fi
104    else
105        trace "Still have $RUNNING arb-processes - terminating nevertheless"
106    fi
107
108    debug "listen_pipe_unlogged waits for subshells ..."
109    wait
110
111    trace "cleaning up arb session"
112    arb_clean show_session
113    arb_clean session
114
115    debug "listen_pipe_unlogged terminates"
116}
117
118if_command() {
119    local CMD=$1; shift
120
121    local FULL_CMD=$($ARBHOME/bin/arb_path.sh -x ${CMD})
122    if [ -n "${FULL_CMD}" ]; then
123        local CALL="${FULL_CMD} $@"
124        echo "{${CALL}}"
125        ${CALL}
126    # else
127        # echo "{no ${CMD}}"
128    fi
129    # never report errors from here
130    true
131}
132
133desktop_info() {
134    echo "XDG_CURRENT_DESKTOP='${XDG_CURRENT_DESKTOP:-}'"
135    echo "GDMSESSION='${GDMSESSION:-}'"
136    # for combinations occurring on different systems see
137    # https://askubuntu.com/questions/72549/how-to-determine-which-window-manager-is-running/227669#227669
138
139    if_command konqueror --version
140    if_command gnome-panel --version
141    if_command xfce4-about --version
142    if_command cinnamon --version
143}
144
145shared_library_dependencies() {
146    case `uname` in
147        Linux)
148            LIST_DYNLIBS="ldd"
149            BINARIES="bin/arb_ntree lib/libARBDB.so lib/libCORE.so lib/libWINDOW.so"
150            ;;
151        Darwin)
152            LIST_DYNLIBS="otool -L"
153            # Darwin ARB links internal stuff static
154            BINARIES="bin/arb_ntree"
155            ;;
156        *)
157            LIST_DYNLIBS="echo UNSUPPORTED_OS "
158            ;;
159    esac
160    for binary in $BINARIES; do
161        echo -e "Library dependencies for $ARBHOME/$binary:"
162        $LIST_DYNLIBS $ARBHOME/$binary
163    done
164}
165
166wrapped_info() {
167    local TAG=$1; shift
168    local CMD=$1; shift
169    echo "--------------------"
170    echo "[$TAG start]"
171    eval $CMD
172    echo "[$TAG end]"
173    echo ""
174}
175
176collect_system_information() {
177    echo "System information"
178    echo ""
179    echo "The information below has been collected by ARB."
180    echo "Please do not publish without being aware that it might contain personal information."
181    echo "[current date: `date`]"
182    echo ""
183
184    local ARB_RELEVANT="| grep -i ARB"
185
186    wrapped_info "version" "$ARBHOME/bin/arb_ntree --help"
187    wrapped_info "environment" "printenv $ARB_RELEVANT"
188    wrapped_info "sina" "$ARBHOME/bin/arb_sina.sh runldd"
189    wrapped_info "OS" "lsb_release -a"
190    wrapped_info "kernel" "uname -mrs ; uname -a ; cat /proc/version"
191    wrapped_info "desktop" "desktop_info"
192
193    wrapped_info "shared libraries" "shared_library_dependencies"
194    wrapped_info "disk" "df -h"
195    wrapped_info "memory" "free -m ; cat /proc/meminfo"
196    wrapped_info "user limits" "ulimit -a"
197    wrapped_info "ARB processes" "ps aux $ARB_RELEVANT"
198
199    wrapped_info "CPU" "cat /proc/cpuinfo"
200    wrapped_info "X server" "xdpyinfo"
201    # wrapped_info "X" "Y"
202}
203
204erase_old_logs() {
205    local LOGBASE=$1
206    if [ -d "$LOGBASE" ]; then
207        # remove files older than 15 days inside and below LOGBASE
208        local OLD=$(( 60 * 24 * 15 ))
209        find $LOGBASE -type f -cmin +$OLD -exec rm {} \;
210        # remove empty directories inside and below LOGBASE
211        find $LOGBASE -type d -depth -empty -mindepth 1 -exec rmdir {} \;
212    fi
213}
214
215listen_pipe() {
216    # this is just a wrapper around listen_pipe_unlogged.
217    # wrapper performs ARB session logging
218    local NAMED_PIPE=$1
219
220    if [ -z ${ARB_PROP:-} ]; then
221        # should never come here, if arb has been started via script 'arb'
222        # (e.g. happens when arb_ntree was started from debugger and then 'start second database' has been called)
223        listen_pipe_unlogged $NAMED_PIPE
224    else
225        local LOGBASE=$ARB_PROP/logs
226        local LOGDIRID=`date '+%Y%m%d_%H%M%S'`.$$
227        local LOGDIR=$LOGBASE/$LOGDIRID
228        local NTREE_STATUS=
229
230        mkdir -p $LOGDIR
231
232        if [ -d "$LOGDIR" ]; then
233            local RUNLOG=$LOGDIR/run.log
234            local SERVERLOG=$LOGDIR/server.log
235            local SYSLOG=$LOGDIR/sys.info
236            local CRASHFLAG=$LOGDIR/crashed
237            local FAILFLAG=$LOGDIR/failed
238
239            # tell arb to start servers as logging daemons
240            export ARB_SERVER_LOG=$SERVERLOG
241            echo "`date` arb server.log created by arb_launcher" > $SERVERLOG
242
243            # forward server output to launcher-tty (non-blocking)
244            tail -f $SERVERLOG &
245            local TAILPID=$!
246
247            ( ( collect_system_information 2>&1 ) > $SYSLOG ; erase_old_logs $LOGBASE ) &
248            ( listen_pipe_unlogged $NAMED_PIPE $LOGDIR ) 2>&1 | tee $RUNLOG
249
250            if [ -e $CRASHFLAG ]; then
251                # only detects crashes of arb_ntree
252                # (clients are not started via arb_launcher and they usually crash when server exits)
253                NTREE_STATUS=crash
254            else
255                if [ -e $FAILFLAG ]; then
256                    NTREE_STATUS=fail
257                fi
258            fi
259
260            if [ "$NTREE_STATUS" != "" ]; then
261                echo "abnormal termination (NTREE_STATUS='$NTREE_STATUS')" >> $RUNLOG
262            else
263                echo "normal termination" >> $RUNLOG
264            fi
265
266            local TARBALLNAME=session.$LOGDIRID.tgz
267
268            debug "killing tail on server-log (pid=$TAILPID)"
269            kill ${TAILPID}
270
271            echo "`date` arb_launcher terminates now. leftover servers may continue logging into this file" >> $SERVERLOG
272            echo "`date` End of log (now archive into $LOGBASE/$TARBALLNAME)" >> $RUNLOG
273
274            ( cd $LOGBASE ; tar -zcf $TARBALLNAME $LOGDIRID )
275            rm -f $RUNLOG $SYSLOG $CRASHFLAG $FAILFLAG
276            rmdir $LOGDIR
277
278            local FULLTARBALL=$LOGBASE/$TARBALLNAME
279            echo ""
280            echo "Session log has been stored in $FULLTARBALL"
281
282            local LATESTLINK=~/ARB_last_session.tgz
283            if [ -h $LATESTLINK ]; then
284                rm $LATESTLINK
285            fi
286            if [ -e $LATESTLINK ]; then
287                echo "$LATESTLINK already exists and is no symlink"
288            else
289                (cd ~; ln -s $FULLTARBALL $LATESTLINK )
290                echo "    and is also accessible via $LATESTLINK"
291            fi
292
293            if [ "$NTREE_STATUS" != "" ]; then
294                echo ""
295                if [ $NTREE_STATUS = "crash" ]; then
296                    echo "ARB crashed :-("
297                    echo "To report this goto http://bugs.arb-home.de/wiki/BugReport"
298                    echo "Please include the session log(s) mentioned above!"
299                    echo ""
300                else
301                    echo "ARB terminated abnormally"
302                fi
303
304                local SECWAIT=
305                echo "[$(date)]"
306                if [ -z "${ARB_SCRIPTED_SECS_WAIT_ON_ERROR:-}" ]; then
307                    local DAYWAIT=3
308                    echo "[press ENTER or wait ${DAYWAIT} days]"
309                    SECWAIT=$((${DAYWAIT}*24*60*60))
310                else
311                    SECWAIT=${ARB_SCRIPTED_SECS_WAIT_ON_ERROR}
312                    echo "[press ENTER or wait ${SECWAIT} seconds]"
313                fi
314                read -t ${SECWAIT} A
315            fi
316            true
317        else
318            echo "Error creating directory '$LOGDIR'"
319            false
320        fi
321    fi
322}
323
324killtree() {
325    local _pid=$1
326    local _sig=${2:-TERM}
327
328    debug "killtree pid=${_pid} with sig=${_sig} pid=$$"
329    kill -stop ${_pid} # stop quickly forking parent from producing childs
330    killchilds ${_pid} ${_sig}
331    kill ${_sig} ${_pid}
332}
333killchilds() {
334    local _pid=$1
335    local _sig=${2:-TERM}
336
337    debug "killchilds pid=${_pid} with sig=${_sig} pid=$$"
338    for _child in $(ps -o pid --no-headers --ppid ${_pid}); do
339        killtree ${_child} ${_sig}
340    done
341}
342
343term_handler() {
344    local NAMED_PIPE=$1
345
346    trace "Killing ARB session for ARB_PID=$ARB_PID"
347    arb_clean session
348    debug "arb_clean done - now killing process tree"
349    killchilds $$ -TERM
350    debug "killchilds done - exiting $$"
351    exit
352}
353
354create_pipe_reader() {
355    local NAMED_PIPE=$1
356    local PARENT_PID=$2
357
358    if [ -z "${ARB_LAUNCHER:-}" ]; then
359        export ARB_LAUNCHER=0
360    else
361        export ARB_LAUNCHER=$(($ARB_LAUNCHER+1))
362    fi
363
364    debug "Creating named pipe '$NAMED_PIPE'"
365
366    # (i did not manage to recover from SIGINT w/o termination of listen_pipe)
367    # => disable SIGINT handler
368    trap '' INT
369    trap "term_handler $NAMED_PIPE" TERM
370    trap "rm -f $NAMED_PIPE" EXIT
371
372    { mkfifo -m 600 $NAMED_PIPE && listen_pipe $NAMED_PIPE ; debug "listen_pipe done" ; } || \
373      { echo "Error creating pipe '$NAMED_PIPE'" ; kill $PARENT_PID ; }
374
375    debug "Pipe reader for '$NAMED_PIPE' terminates.."
376    rm -f $NAMED_PIPE
377    debug "Pipe '$NAMED_PIPE' removed"
378}
379
380initial_send_to_launcher() {
381    local NAMED_PIPE=$1
382    local CMD=$2
383
384    send_to_launcher $NAMED_PIPE "$CMD"
385
386    # now allow pipe reader to terminate:
387    send_to_launcher $NAMED_PIPE "allow_termination"
388}
389
390wait_for_pipe() {
391    local NAMED_PIPE=$1
392
393    while [[ ! -p $NAMED_PIPE ]];
394      do
395      echo "Waiting for '$NAMED_PIPE'.."
396      sleep 1
397    done
398    debug "pipe is open"
399}
400
401get_pipe_name() {
402    local SOCKETDIR="$HOME/.arb_tmp/sockets"
403    mkdir -p "$SOCKETDIR"
404    chmod 0700 "$SOCKETDIR"
405    echo "$SOCKETDIR/arb_launcher.$ARB_PID"
406
407    # instead of the above code, use the following to test a pipe-creation failure:
408    # echo "/arb_launcher.$ARB_PID"
409}
410
411launcher() {
412    local ASYNC=0
413    if [ "$1" = "--async" ]; then
414        ASYNC=1
415        shift
416    fi
417    local CMD="$*"
418
419    if [ -z "$ARB_PID" ]; then
420        echo "Error: environment variable ARB_PID is unset. terminating.."
421        false
422    else
423        if [ -z "$1" ]; then
424            echo "Usage: arb_launcher \"shellcommand\""
425            echo ""
426            echo "          runs 'shellcommand'"
427            echo "          "
428            echo "          The initial call to arb_launcher will block until 'shellcommand' terminates."
429            echo ""
430            echo "          Subsequent calls will not block. They are started from the context of the"
431            echo "          initial call. The initial call will wait for all started commands."
432            echo ""
433            echo "       arb_launcher \"TERMINATE\""
434            echo ""
435            echo "          terminate the launcher without waiting for spawned commands."
436            echo ""
437        else
438            debug "Using ARB_PID '$ARB_PID'"
439            local NAMED_PIPE=$(get_pipe_name)
440            debug "Using NAMED_PIPE '$NAMED_PIPE'"
441
442            if [[ ! -p $NAMED_PIPE ]]; then
443                ( wait_for_pipe $NAMED_PIPE ; initial_send_to_launcher $NAMED_PIPE "$CMD" ) &
444                if (( $ASYNC==1 )); then
445                    create_pipe_reader $NAMED_PIPE $$ &
446                else
447                    create_pipe_reader $NAMED_PIPE $$
448                fi
449            else
450                debug "pipe already was open"
451                send_to_launcher $NAMED_PIPE "$CMD"
452            fi
453
454            # if pipe-reader was started from current process
455            # -> blocks until all launched processes have terminated
456            if (( $ASYNC==0 )); then
457                wait
458            fi
459        fi
460    fi
461}
462
463launcher "$@"
464debug "arb_launcher exits!"
465
Note: See TracBrowser for help on using the repository browser.