szaydel
5/28/2014 - 11:53 AM

RSF1 rsfexec script with BrickstorOS-specific modifications. Mainly, changes are to network handling, where ifconfig, which by nature is non

RSF1 rsfexec script with BrickstorOS-specific modifications. Mainly, changes are to network handling, where ifconfig, which by nature is non-persistent is replaced with ipadm, which by nature is persistent, but we use -t throughout, in order to make it non-persistent, which is what we want with floating interfaces.

#!/bin/sh
# $Id: rsfexec,v 1.104 2014/02/26 16:09:03 matt Exp $
#
# Script:   rsfexec
#
# Description:  Overall handler for service scripts
#
# Platform: Unix
#
# Author:   High-Availability.Com Ltd
#

# --- begin redirect to log for anything to stdout or stderr

exec 1> /tmp/rsfexec.log
exec 2>&1

set -o xtrace

. /opt/HAC/bin/rsf.sh

t_out=/opt/HAC/RSF-1/bin/timeout.sh
fping=/opt/HAC/bin/fping
fping6=/opt/HAC/bin/fping6

PRODUCT_RC_DIR=${PRODUCT_DIR}"/etc"

script="`basename $0`"  # for dated_echo()

# Some versions of OpenSolaris have broken ifconfig so that it is not
# re-entrant. Try to fix that by locking calls to ifconfig. (This only
# locks calls by RSF-1... Other users or programmes running ifconfig
# can still cause problems.
if test -f ${PRODUCT_BIN}/with_lock; then
    IFCONFIG="${PRODUCT_BIN}/with_lock /tmp/RSF-1_ifconfig_lock ifconfig"
    IPADM_CMD="${PRODUCT_BIN}/with_lock /tmp/RSF-1_ifconfig_lock /usr/sbin/ipadm"
else
    IFCONFIG=ifconfig
    IPADM_CMD=/usr/sbin/ipadm
fi

#
# Check and create broken log directory if required.
if [ ! -z "${BROKEN_LOG}" -a ! -d "${BROKEN_LOG}" ] ; then
    mkdir -p "${BROKEN_LOG}"
fi

#
# If you need to ask...
usage()
{
    echo "First set environment variables with extra configuration info, then:"
    echo "${script} [start|stop|panic] <service> <interface> <floatname> <attempt> <options> <floatname_netmask> <floatname_address_type>"
}

#
# If fping is available, use it to check the service floating IP address
# is free.  If not available, give the "couldn't ping" result.
#
# Returns 0 if it could ping, otherwise 1.
#
# IFSTAT -t tests to see if the passed address is enabled local on the
# machine, returning 0 if it does, otherwise 1.

ping_ip_address()
{
    IFSTAT=/opt/HAC/bin/ifstat
    addr="$1"
    atype="${2:-inet}"

    dated_echo "ping_ip_address: checking ${atype} ${addr} does not exist on the network"

    if match "${atype}" "inet6"; then
    if [ -x ${fping6} ] ; then
        if [ ! -e ${t_out} ] || [ ! -x ${t_out} ]; then
        dated_echo "error: ${t_out} not found";
                exit $RSF_ABORT
        else
        if "${t_out}" 5 "${fping6}" -q -t200 2>/dev/null "${addr}" ; then
                    return 0
        fi
        fi
    else
        if [ "${COMPANY_OS}" = "sunos" ]; then
        if ping -A inet6 "${addr}" &>/dev/null ; then
            return 0
        fi
        fi
    fi
    else
    if [ -x ${IFSTAT} ] ; then
        if ${IFSTAT} -t ${addr} ; then
        dated_echo "Address ${addr} is local and UP, skipping address test (output of ifconfig/netstat below)"
        ifconfig -a
        netstat -rn
        return 1
        fi

        if [ -x ${fping} ] ; then
        if ${fping} -q -t200 2>/dev/null ${addr} ; then
            return 0
        fi
        fi
    fi
    fi

    dated_echo "Address ${addr} is not in use, OK to start services."
    return 1
}

do_fping_check()
{
    # Check that our floating IP address (if any) is free.
    # Just go broken_safe if the address is already in use.
    if skip_net_check "${ipface}"; then
    : no address to check
    else
    if ping_ip_address "${floatname}" "${floatname_type}"; then
        dated_echo "Service ${service} floating address ${floatname} in use,"
        dated_echo "marking ${service} broken_safe."
        dated_echo "Floating address ${floatname} in use." > ${BROKEN_LOG}/${service}.log
            dated_echo "ARP table dump follows:"
            arp -an
        exit ${RSF_SAFE}
    fi
    fi
}

#
# Function:
#   skip_net_device()
#
# Description:
#   Check some special 'interface names' for strings which need to
#       inhibit ifconfig, & return true if a match is found.
#   Note 'true' is zero, as this is treated as an exit code. This
#       check is done when we are just about to plumb the address in.

# Parameters:
#   1) Network device name, of which a subset is picked up here [none].
#
skip_net_device()
{
    case "$1" in
    [Nn][Oo][Nn][Ee])     return 0  ;;
    esac
    return 1
}

#
# Function:
#   skip_net_check()
#
# Description:
#   Different case from skip_net_device, here this test is for fping
#   checking to make sure the IP address does not exist on the net.
#   *HOWEVER* if the device plumbing is to be deferred then we still
#   would like to check that the IP address is not out there on the LAN
#   before starting services.
# Parameters:
#   1) Network device name, of which a subset is picked up here [none].
#
skip_net_check()
{
    case "$1" in
    [Nn][Oo][Nn][Ee])     return 0  ;;
    esac
    return 1
}


# function args ipdev "type address netmask"...
# set ipdev global variable
# For each "type address netmask" triple:
#  set floatname_type=type
#  set floatname=address
#  set netmask=netmask
#  call function with args
#
do_extra_ipdev_list3()
{
    do_extra_ipdev_list_function="$1"
    do_extra_ipdev_list_args="$2"
    ipface="$3"

    shift
    shift
    shift
    while match x"$1" 'x?*'; do
    floatname_type="$1"
    floatname="$2"
    netmask="$3"
    match x"$3" x"-" && netmask=""
    shift
    shift
    shift
    $do_extra_ipdev_list_function $do_extra_ipdev_list_args
    done
}


# Usage: foreach_network_link function args...
# Find all network interfaces and addresses for this service, from:
#  1) The original IPDEVICE keyword, and floatname in the SERVICE line
#  2) Any ADDITIONAL_IP_n environment variables
#  3) Any EXTRA_IPDEVICE and IPDEV_ADDRESS config file lines, passed to
#     rsfexec as RSF_IPDEV_n environment variables
# For each one, set global variables floatname, floatname_type, ipface
#  then call the given function with the given arguments
foreach_network_link()
{
    foreach_network_link_function="$1"
    shift

    # First do the original IPDEVICE line, with the SERVICE line floatname
    # passed to rsfexec as command line arguments
    floatname="$main_floatname"
    floatname_type="$main_floatname_type"
    ipface="$main_ipface"
    netmask="$config_netmask"

    ${foreach_network_link_function} "$@"

    # Cycle through ADDITIONAL_IP_n environment variables
    foreach_netlink_varnumber=0;
    while true; do
    foreach_netlink_varnumber=`expr ${foreach_netlink_varnumber} + 1`
    foreach_netlink_varname="ADDITIONAL_IP_${foreach_netlink_varnumber}"
    eval value=\$${foreach_netlink_varname}
    match x"${value}" x && break
    floatname_type="inet"
    split "${value}" floatname ipface
    ${foreach_network_link_function} "$@"
    # # Use of ADDITIONAL_IP_n is deprecated, succeed with warnings.
    # if match "$rsfexec_success_exit_code" "${RSF_OK}"; then
    #     dated_echo "Warning: use of ADDITIONAL_IP_n is deprecated, use"
    #     dated_echo "floatname NONE, then EXTRA_IPDEVICE & IPDEV_ADDRESS"
    #     rsfexec_success_exit_code="${RSF_WARN}"
    # fi
    done

    # Process EXTRA_IPDEVICE and IPDEV_ADDRESS lines in the config file,
    # passed as RSF_IPDEV_n environment variables.
    # RSF_IPDEV_n interface (protocol address mask-or-prefix)...
    # eg. RSF_IPDEV_1 bge0 inet 192.168.1.2 - inet6 fe80::b0c3:b5ff:fe14:33a 64
    foreach_netlink_varnumber=0;
    while true; do
    foreach_netlink_varnumber=`expr ${foreach_netlink_varnumber} + 1`
    foreach_netlink_varname="RSF_IPDEV_${foreach_netlink_varnumber}"
    eval value=\$${foreach_netlink_varname} # interface (type addr mask)...
    match x"${value}" x && break
    do_extra_ipdev_list3 "${foreach_network_link_function}" "$*" ${value}
    done
}

# Check if we should be doing this operation now, and if so loop round
# all network interfaces and addresses, setting ipface and floatname
# variables and calling do_netif for each one.
# Return true=0 if operations were done, false=1 if not.
do_all_netif()
{
    when="$1"      # When this procedure is being called. [before|after]
    type="$2"      # The type of call. [up|down|none]
    shift
    shift
    opts="$*"      # Options string from the config file
    doit="no"      # Default action is to do nothing. [no|yes]

    # Defaults bring ip up before and down after services.
    ipup="before"  ; match "$opts" "*ip_up_after*"    && ipup="after"
    ipdown="after" ; match "$opts" "*ip_down_before*" && ipdown="before"

    #
    # Process and set doit, possible values for ${when}_${type} are:
    #
    #   before_up
    #    after_up
    #   before_down
    #    after_down
    #      now_up
    #      now_down
    #
    case "${when}_${type}" in
    ${ipup}_up) doit="yes"  ;;
    ${ipdown}_down) doit="yes"  ;;
    now_up)     doit="yes"  ;;
    now_down)   doit="yes"  ;;
    esac

    # Uncomment for debugging purposes
    #echo "debug do_netif: doit=\"${doit}\" when=\"${when}\" type=\"${type}\" opts=\"${opts}\""

     match "${doit}" "yes" || return 1
     foreach_network_link do_netif "$@"
     return 0
}

#
# Function:
#   do_netif()
#
# Description:
#   Determines if the interface change function (netif) should be
#   called, which is decided on if it's down or up and at what stage of
#   services startup the Virtual IP address (VIP) should be introduced
#   (before or after service startup). Parameters:
#   1) When is the function being called, [before|after] scripts have run.
#   2) The type of call (note value none is for panic). [up|down|none]
#   3) New options from the config file via rsfmon. Currently valid options
#      recognised by this function are:
#       ip_up_after    - bring up the VIP after services have started
#       ip_down_before - take down the VIP before services have started
#
do_netif()
{
    # Check whether ip interface is to be ignored, & issue command if not
    skip_net_device "${ipface}" || netif ${type}
}

#
# Function:
#   ipv6_init()
#
# Description:
#   Checks the floatname address type to determine if the floatname is an
#   IPv6 address. If the floatname address type is "inet6" then the network
#   interface ${ipface} is checked to see if it has IPv6 plumed and plums it
#   in if it doesn't.
#
#   If the floatname address type is found to be 'inet'. If RSF-1 does not
#   support IPv6 for the OS/platform specified in ${COMPANY_OS} the function
#   returns 1 (false), otherwise the function returns 0 (true).
#
ipv6_init()
{
    case "${COMPANY_OS}" in
    'sunos')
        if [ ${_action} == "up" ] && [ `ifconfig -a6 | grep $ipface | wc -l` -eq 0 ]; then
        dated_echo "IPV6 not plumbed on ${ipface} - PLUMBING NOW ..."
        dated_echo "${IFCONFIG} ${ipface} inet6 plumb up"
        ${IFCONFIG} ${ipface} inet6 plumb up
        fi
        ;;
    *)
        dated_echo "error: RSF-1 does not support IPv6 on (${COMPANY_OS})"
        return 1
        ;;
    esac
    return 0
}

#
# Function:
#   ipv6_unplumb_check()
#
# Description:
#   If it is and there are no more services configured to use an IPv6
#   address on the network interface ${ipface} then IPv6 is unplumed
#   from the network interface. If the interface does not require IPv6
#   to be unplumed the function returns 1 (false). If RSF-1 does not
#   (yet) support IPv6 for the OS/platform specified in ${COMPANY_OS}
#   the function returns 2 (false), otherwise the function returns 0
#   (true)
#
ipv6_unplumb_check()
{
#if inet_flag="inet6"; then
    if match "${floatname_type}" "inet6"; then
    case "${COMPANY_OS}" in
        'sunos')
        if [ `ifconfig -a6 | grep $ipface | wc -l` -eq 1 ]; then
            dated_echo "No more IPv6 services on ${ipface} - UNPLUMBING NOW..."
            ${IFCONFIG} ${ipface} inet6 unplumb
        fi
        return 0
        ;;
        *)
        dated_echo "error: IPv6 is not supported on this platform (${COMPANY_OS})"
        return 2
        ;;
    esac
    fi
    return 1
}

#
# manipulate (floating) network interface
#

netif()
{
    set -o xtrace
    _action=$1

    #
    # First of all set command variables that perform the
    # platform-dependent interface configuration. These variables are
    # used later (in this function) to actually manipulate the
    # interface/routing table on the node, they are:
    #
    #   ifup        - plumb/enable/bring-up a VIP on an interface
    #   ifdown      - unplumb/disable/bring-down a VIP on an interface
    #   ifargs      - arguments to be used in addition to ${ifup}
    #   route       - extras required to add address to routing table
    #   routedown   - extras required to remove address from routing table
    #
    if match "X${netmask}" X || match "X${netmask}" "X-"; then
    set_netmask=""
    else
    set_netmask="-N${netmask}"
    fi

    #
    # If action is up, test for IPv6 and prepare interface for
    # configuation if required
    #
    if match "${floatname_type}" "inet6"; then
    ipv6_init || exit ${RSF_ABORT}
    fi

    ifargs=""
    case "${COMPANY_OS}" in
    'sunos'|'brickstoros')
        if match x"${floatname_type}" x"inet6"; then
        match x"${netmask}" x || match "X${netmask}" "X-" || ifargs="/${netmask}"
        else
        if match "X${netmask}" X || match "X${netmask}" "X-"; then
            ifargs=' netmask + broadcast +'
        else
            ifargs=" `hac_getmask -N${netmask} -bn ${floatname}`"
        fi
        fi

        case "${ipface}" in
        *,*)
            # For earlier Solaris versions where native IPMP
            # NAFO functionality is required. Solaris 10
            # transparently redirects IPMP NAFO group ifconfig
            # commands correctly ipface contains comma separated
            # IPMP NAFO group network interfaces
            ;;

        *)
            # BrickstorOS should fall into this case and this is where brunt of the
            # network plumbing is prepared for later execution.
            cidr_netmask=$(mask_to_cidr ${netmask}) # Converts netmask from xxx.xxx.xxx.xxx to /XX.
            # We need to make sure to drop anything along the lines of `-|_|.`, since ipadm does not
            # handle well ifname/some-string and instead wants ifname/somestring.
            xx=$(/usr/bin/awk '{gsub(/\-|\_|\./, ""); \
                printf("%s", substr($0, 0, 8))}' < <(echo ${floatname}))
            ipadm_tag="${xx}${cidr_netmask}"

            if match "${ipface}" "*:*" ; then
            ${IFCONFIG} ${ipface} plumb # needed for Solaris 8+
            # This block is zone related, we should not reach it.
            if match "X$opt_zone" X; then
                    ifup="${IFCONFIG} ${ipface} inet ${floatname}${ifargs} up"
            else
                    ifup="${IFCONFIG} ${ipface} inet ${floatname}${ifargs} up zone ${opt_zone}"
            fi
            ifdown="${IFCONFIG} ${ipface} unplumb"
            else
            # We do not run in a zone, so should always hit these lines. This is where
            # we do work necessary to configure a floating VIP using ipadm.
            # Conversion from ifconfig to ipadm means we are not using temporary interfaces
            # and addresses explicitly. Whereas, with ifconfig by default interfaces are temporary,
            # with ipadm interfaces are persistent. Also, this will not work if there is not already
            # an existing interface object, which critically must have been created with a `-t` flag.
            # If an interface object does not exist, this will fail, and service will be broken_safe.
            # If monitoring of interfaces is enabled in the config, any missing interfaces will put
            # services into blocked state. But, if we are not monitoring, we may be in a situation
            # where we try to set an ip on a non-existing iface. This be bad, so we have to account
            # for that here.
            if match "X$opt_zone" X; then
                ifup="{ ${IPADM_CMD} show-if -p -oifname ${ipface} >/dev/null 2>&1 || ${IPADM_CMD} create-if -t ${ipface};
                ${IPADM_CMD} create-addr -t -T static -a local=${floatname}/${cidr_netmask} ${ipface}/${ipadm_tag}; }"
            else # Leaving this as is, since we will not use zones.
                    ifup="${IFCONFIG} ${ipface} addif ${floatname}${ifargs} up zone ${opt_zone}"
            fi
            ifdown="${IPADM_CMD} delete-addr ${ipface}/${ipadm_tag}"
            fi
            route=':'
            routedown=':'
            ;;

        esac
        ;;
    'linux')
        ifargs="`hac_getmask ${set_netmask} -bn ${floatname}`"
        ifup="ifconfig ${ipface} inet ${floatname} ${ifargs} up; hac_grarp ${ipface} ${floatname}"
        ifdown="ifconfig ${ipface} down"
        route="route add -host ${floatname} dev ${ipface}"
        routedown=':'
        ;;
    'aix')
        ifargs="`hac_getmask ${set_netmask} -bn ${floatname}`"
        ifup="ifconfig ${ipface} alias ${floatname} ${ifargs}"
        ifdown="ifconfig ${ipface} delete ${floatname}"
        route=':'
        routedown=':'
        ;;
    'hp-ux')
        ifargs="`hac_getmask ${set_netmask} -bn ${floatname}`"
        ifup="ifconfig ${ipface} ${floatname} ${ifargs}"
        ifdown="ifconfig ${ipface} down; ifconfig ${ipface} 0.0.0.0"
        route=':'
        routedown=':'
        ;;
    'freebsd'|'openbsd'|'darwin')
        ifargs="`hac_getmask ${set_netmask} -bn ${floatname}`"
        # ifconfig alias on FreeBSD gives spurious error anyway
        ifup="ifconfig ${ipface} inet ${floatname} ${ifargs} alias >/dev/null 2>&1; true"
        ifdown="ifconfig ${ipface} inet ${floatname} -alias"
        route="route add ${floatname} -interface ${ipface}"
        routedown="route delete ${floatname}"
        ;;
    *)
        ifup='dated_echo "*** ERROR - Unrecognised platform: ${COMPANY_OS}"'
        ifdown=${ifup}
        route=':'
        ;;
    esac

    #
    #
    case "${_action}" in

    'up')
        # For earlier Solaris versions where native IPMP NAFO
        # functionality is required. Solaris 10 transparently
        # redirects IPMP NAFO group ifconfig commands correctly
        if [ "${COMPANY_OS}" = 'sunos' ]; then
        case "${ipface}" in

            *,*)
            # ipface contains comma separated IPMP NAFO group
            # network interfaces. If all network interfaces in
            # IPMP NAFO group fail to configure then abort
            # service startup.
            ipmp_group="abort"
            for ipface_ipmp in `echo "${ipface}" | sed 's/,/ /g'`
            do
                eval "${IFCONFIG} ${ipface_ipmp} addif ${floatname}${ifargs} up"
                if [ $? -ne 0 ]; then
                dated_echo "*** WARNING - failed to configure ${ipface_ipmp} network interface in IPMP NAFO group ***"
                else
                dated_echo "*** NOTICE - configured ${ipface_ipmp} network interface in IPMP NAFO group ***"
                ipmp_group="OK"
                break
                fi
            done

            if [ X"${ipmp_group}" = X'abort' ]; then
                dated_echo "*** ERROR - failed to configure any network interfaces in IPMP NAFO group, aborting ***"
                exit ${RSF_ABORT}
            fi
            ;;

        esac
        fi

        if [ X"${ipmp_group}" = X'OK' ]; then
        : IPMP NAFO group network interface already configured
        else
        eval ${ifup}
        if [ $? -ne 0 ]; then
            dated_echo "*** ERROR - failed to configure ${ipface} network interface, aborting ***"
            notify_watchers "${LOG_CRIT}" "RSF_NETIF_UP_FAIL" "ipface=${ipface}" "vip=${floatname}"
            exit ${RSF_ABORT}
        else
            eval ${route}
            notify_watchers "${LOG_INFO}" "RSF_NETIF_UP" "ipface=${ipface}" "vip=${floatname}"
        fi
        fi
        #
        # Log our results.
        dated_echo '========= ifconfig after interface/vip plumbing ========='
        echo
        ifconfig -a
        echo
        dated_echo '========= netstat after  interface/vip plumbing ========='
        echo
        netstat -rn
        echo
        dated_echo '=========      ifconfig/netstat complete        ========='

        #
        # Tell everybody we are finished.
        notify_watchers "${LOG_INFO}" "RSF_SERVICE_STARTUP_COMPLETE" "service=${service}"
        ;;

    'down')
        # For earlier Solaris versions where native IPMP NAFO
        # functionality is required. Solaris 10 transparently
        # redirects IPMP NAFO group ifconfig commands correctly
        if [ "${COMPANY_OS}" = 'sunos' ]; then
        case "${ipface}" in

            *,*)
            # ipface contains comma separated IPMP NAFO group
            # network interfaces
            ipmp_group="OK"
            for ipface_ipmp in `echo "${ipface}" | sed 's/,/ /g'`
            do
                eval "${IFCONFIG} ${ipface_ipmp} removeif ${floatname}"
                if [ $? -eq 0 ]; then
                dated_echo "*** NOTICE - unconfigured ${ipface_ipmp} network interface in IPMP NAFO group ***"
                break
                fi
                dated_echo "*** WARNING - failed to unconfigure ${ipface_ipmp} in IPMP NAFO group ***"
            done
            ;;

        esac
        fi

        notify_watchers "${LOG_INFO}" "RSF_NETIF_DOWN" "ipface=${ipface}" "vip=${floatname}"

        if [ X"${ipmp_group}" = X'OK' ]; then
        : IPMP NAFO group network interface already unconfigured
        else
        eval ${ifdown}
        eval ${routedown}
    #
    # Check number IPv6 services - unplumb IPv6 from the interface if no more services
    #

        ipv6_unplumb_check

        fi
        ;;

    *)
        dated_echo "Invalid ${ipface} action, ${_action}, aborting."
        exit ${RSF_ABORT}
        ;;

    esac
}


#
# run service scripts
#
run_scripts()
{
    service="$1"
    svc_state="$2"
    attempt="$3"
    options="$4"
    netif_done="no"

    if [ -f "${PRODUCT_RC_DIR}/${service_dir}" ]
    then
        dated_echo "${PRODUCT_RC_DIR}/${service_dir} is not a directory."
        dated_echo "*** Failed!  ABORTING ***"
        exit ${RSF_ABORT}
    fi

    if [ ! -d "${PRODUCT_RC_DIR}/${service_dir}" ]
    then
        dated_echo "No service directory, ${PRODUCT_RC_DIR}/${service_dir}."
        dated_echo "*** Failed!  ABORTING ***"
        exit ${RSF_ABORT}
    fi

    #
    # Check for "After" file to invert when interfaces are changed
    if [ -f "${PRODUCT_RC_DIR}/${service_dir}/IPAfter" ];   then
        dated_echo "Ignoring ${PRODUCT_RC_DIR}/${service_dir}/IPAfter"
        dated_echo "This has been replaced by config file service OPTIONs."
    fi

    cd ${PRODUCT_RC_DIR}/${service_dir}

    case "${svc_state}" in

    'start')
        elist=`/bin/ls -A S* 2>/dev/null | env _POSIX2_VERSION=1 sort -n +0.1`
        netif_type="up"
        ;;

    'stop')
        elist=`/bin/ls -A K* 2> /dev/null | env _POSIX2_VERSION=1 sort -n +0.1`
        netif_type="down"
        ;;

    'panic')
        elist=`/bin/ls -A P* 2> /dev/null | env _POSIX2_VERSION=1 sort -n +0.1`
        netif_type="none"
        ;;

    *)
        dated_echo "Invalid service state, ${svc_state}."
        return 1
        ;;

    esac

    # perform any network interface manipulation *before* scripts run
    do_all_netif before ${netif_type} "$options" && netif_done="yes"
    for svc_script in ${elist}; do

        # check for executable bit
        dated_echo "Running ${svc_script} ${svc_state} ${attempt}"
        notify_watchers "${LOG_INFO}" "RSF_EXEC_SCRIPT" "script=${svc_script}" "state=${svc_state}" "attempt=${attempt}"

        if [ -x "${svc_script}" ]; then
            # assume non-sh
            ./${svc_script} ${svc_state} ${attempt}
            ret_stat=$?
        else
            sh ${svc_script} ${svc_state} ${attempt}
            ret_stat=$?
        fi

        case ${ret_stat} in

        ${RSF_OK})
            ;;

        ${RSF_WARN})
            dated_echo "*** ${svc_script} returned warnings! Continuing ***"
            ;;

        ${RSF_RESTART})
            dated_echo "*** ${svc_script} failed!  RESTARTING ***"
            exit ${RSF_RESTART}
            ;;

        ${RSF_ABORT})
            dated_echo "*** ${svc_script} failed!  ABORTING ***"
            exit ${RSF_ABORT}
            ;;

        ${RSF_SAFE})
            dated_echo "*** ${svc_script} failed BROKEN_SAFE! ***"
            exit ${RSF_SAFE}
            ;;

        ${RSF_UNSAFE})
            dated_echo "*** ${svc_script} failed BROKEN_UNSAFE! ***"
            exit ${RSF_UNSAFE}
            ;;

        ${RSF_PLUMB_IPNOW})
            if match "$netif_done" "no"; then
                do_all_netif now ${netif_type} "$options"
                netif_done="yes"
            fi
            ;;

        *)
            dated_echo "*** ${svc_script} Unrecognised exit status ${ret_stat}! ABORTING ***"
            exit ${RSF_ABORT}
            ;;

        esac
    done

    # perform any network interface manipulation *after* scripts run
    if match "$netif_done" "no"; then
        do_all_netif after ${netif_type} "$options"
        netif_done="yes"
    fi
}

#
# Hook to standby LUNS.
#
standby_luns()
{
    LUN_SERVICE=$1
    ALUAADM="/usr/demo/comstar/bin/aluaadm"
    LUNS_TO_STANDBY="/opt/HAC/RSF-1/etc/${LUN_SERVICE}.standby-luns"

    if [ -f "${LUNS_TO_STANDBY}" ] ; then
        dated_echo "standby_luns - ${LUN_SERVICE}: LUNS located in ${LUNS_TO_STANDBY}"
        if [ -x "${ALUAADM}" ] ; then
            while read lu_to_standby ; do
                dated_echo "standby_luns - ${LUN_SERVICE}: setting lun ${lu_to_standby} to standby"
                ${ALUAADM} standby "${lu_to_standby}"
            done < "${LUNS_TO_STANDBY}"
        else
            dated_echo "standby_luns - ${LUN_SERVICE}: could not locate ${ALUAADM}"
        fi
    else
        dated_echo "standby_luns - ${LUN_SERVICE}: no LUNS requiring standby, ${LUNS_TO_STANDBY} does not exist."
    fi
}

#
# Main
#
if [ $# -lt 5 ] # may be > 5 for rsfmon bounce mod
then
    # incorrect args
    usage
    exit ${RSF_ABORT}
fi

state="$1"
service="$2"
main_ipface="$3"
main_floatname="$4"
attempt="$5"
options="$6"
match x"$options" "x-" && options=""
config_netmask="$7"
# $8 is the address type of main_floatname, & should be "inet" or "inet6"
main_floatname_type="${8:-inet}"

# If all went well exit with this code (at the end of the file).
# To exit with warnings set rsfexec_success_exit_code="${RSF_WARN}"
rsfexec_success_exit_code="${RSF_OK}"

# For each name=value option, set the environment variable opt_name=value
for opt in $options; do
    eval `echo "$opt" | sed -n '/=/s/^/opt_/p;s/^\([^=]*\)=.*$/export \1/p'`
done

if match x"${opt_sdir}" x; then
    # Use a dedicated (rc.servicename.d) directory
    service_dir="rc.${service}.d"
else
    # Use a common (rc.dirname.c) directory
    service_dir="rc.${opt_sdir}.c"
fi


# export args to environment
RSF_SERVICE="${service}"
RSF_IPDEV="${main_ipface}"
RSF_FLOATNAME="${main_floatname}"
#RSF_FLOATNAME_TYPE="${main_floatname_type}"
export RSF_SERVICE RSF_IPDEV RSF_FLOATNAME
# RSF_FLOATNAME_TYPE


case "${state}" in
    'start')
    # Check that our floating IP addresses (if any) are free.
    # Just go broken_unsafe if an address is already in use.
    foreach_network_link do_fping_check
    ;;

    'stop')
        # standby_luns "${service}"
    ;;

    'panic')
    ;;

    *)
    usage
    exit ${RSF_ABORT}
    ;;
esac

STARTTIME=`${DATE}`
run_scripts "${service}" "${state}" "${attempt}" "${options}"
ENDTIME=`${DATE}`

dated_echo "Total run time for service ${state}: `expr ${ENDTIME} - ${STARTTIME}` seconds"

exit ${rsfexec_success_exit_code}