szaydel
8/23/2017 - 9:59 PM

[hiavd systemd files] Service management files necessary to run hiavd under systemd #tags: hiavd, brickstor ha, brickstor-ha, high availabil

[hiavd systemd files] systemd service management files necessary to run hiavd under systemd #tags: hiavd, brickstor ha, brickstor-ha, high availability

hiavd-systemd.sh
content_copyfile_download
#! /bin/bash 
# Copyright 2009-2018 RackTop Systems Inc. and/or its affiliates.
# http://www.racktopsystems.com
#
# The methods and techniques utilized herein are considered TRADE SECRETS
# and/or CONFIDENTIAL unless otherwise noted. REPRODUCTION or DISTRIBUTION
# is FORBIDDEN, in whole and/or in part, except by express written permission
# of RackTop Systems.
#
# @@DESCRIPTION@@  Wrapper Script for hiavd High Availability Daemon
# @@NAME@@         hiavd-systemd.sh
# @@STABILITY@@    unstable
# @@VERSION@@      1.0.4
USER=root
GROUP=root
BINDIR=/usr/racktop/sbin
CMD=${BINDIR}/hiavd
CONFDIR=/etc/racktop
CERTDIR=${CONFDIR}/certs
RUNDIR=/run/racktop
WORKDIR=${CONFDIR}/hiavd
SSHKEYDIR=/etc/ssh/${USER}
PIDFILE=${RUNDIR}/hiavd.pid

#
# Place this script in $BINDIR, which is where the systemd Service Manager
# is going to expect to find it, based on the service configuration manifest.

#
# preflight: Checks for basic requirements without which we cannot run
#
function preflight {
    # Check for desired or required executables
    # The ipmitool check was originally a requirement, but seems like
    # witness for the moment does not need to have this tool. This 
    # requirement may change in the future however.
    printf "%s\n" "[INFO] Required Executable Checks" ;
    [ ! -x /usr/bin/ipmitool ] && {
        printf "HA Witness: Missing ipmitool program.\n" >&2 ;
    }

    [ ! -x ${CMD} ] && {
        printf "HA Witness: <FATAL> Missing hiavd program.\n" >&2 ;
        return 1;
    }

    printf "%s\n" "[INFO] SSH Configuration Check" ;
    # Check for required ssh bits
    [[ ! -d ${SSHKEYDIR} || ! -f ${SSHKEYDIR}/id_rsa ]] && \
    {    
        printf "HA Witness: Missing ssh Directory or Keyfile.\n" >&2 ;
        printf "Please make sure ${SSHKEYDIR} contains private ssh key 'id_rsa'.\n" >&2 ;
        return 1;
    }
    
    printf "%s\n" "[INFO] Service Configuration Check" ;

    return 0
}

#
# start: Responsible for starting service
#
function start
{
    # If we failed pre-flight checks, chances are we won't start.
    if ! preflight ; then return 1; fi

    # This directory is required and may be missing on newly prepared systems.
    [[ ! -d ${CONFDIR} || ! -d "${CONFDIR}/hiavd" ]] && \
        {
            printf "%s\n" \
                "[INFO] Brand New system, creating required directories" ;
            mkdir -p "${CONFDIR}/hiavd" || return 1
            mkdir -p "${CONFDIR}/hiavd/certs" || return 1
            printf "%s\n" \
                "[INFO] Witness is unconfigured, make sure to configure cluster"
        }


    # This directory is required, but only at runtime, so we can just mkdir it.
    [ ! -d ${RUNDIR} ] && mkdir ${RUNDIR}

    # If $PIDFILE exists, we are either running already, in which case
    # we need to not start another instance, or we are not running but
    # previous exit was abnormal and pid file remained.
    if [ -f ${PIDFILE} ]; then
        if pgrep -F ${PIDFILE} > /dev/null 2>&1 ; then       
            printf "HA Witness: Service Running, only single instance allowed.\n" >&2
            return 1
        else
            printf "HA Witness: PID file exists but stale, removing.\n" >&2
            rm ${PIDFILE}
        fi
    fi
    printf "HA Witness: Starting Service\n"
    ${CMD} \
    -c ${WORKDIR}/hiavd.conf \
    -w ${WORKDIR} \
    -pid ${PIDFILE} # Run actual daemon command
}

#
# stop: Responsible for stopping service
#
function stop
{
    if [ ! -f ${PIDFILE} ] || ! pgrep -F ${PIDFILE} > /dev/null 2>&1; then
        printf "HA Witness: Stopping Service, but nothing to do, maybe service dead?\n" >&2
        return
    fi
    printf "HA Witness: Stopping Service (PID = %d)\n" `cat ${PIDFILE}`
	# We assume that the service cleans-up its own pid file when sent
    # a SIGTERM signal.
    pkill -F ${PIDFILE}
 }
 
#
# status: Responsible for obstaining status of service
#
function status
{ 
    if pgrep -F ${PIDFILE} > /dev/null 2>&1 ; then 
        printf "HA Witness: Service Alive!\n"
    else
        printf "HA Witness: Service appears dead!\n"
    fi
}

case "$1" in
	start)
		start
		;;
	stop)
		stop
		;;
	reload)
		stop
		sleep 1
		start
		;;
	status)
		status
		;;
	*)
	echo "Usage: $0 {start | stop | reload | status}"
	exit 1
	;;
esac
exit 0
hiavd.service
content_copyfile_download
# /etc/systemd/system/hiavd.service

[Unit]
Description = RackTop Systems High Availability Daemon (hiavd)
After = syslog.target network.target

[Service]
Type                = simple
ExecStart           = /usr/racktop/sbin/hiavd-systemd.sh start
ExecStop            = /usr/racktop/sbin/hiavd-systemd.sh stop
ExecReload          = /usr/racktop/sbin/hiavd-systemd.sh reload
ProtectHome         = true
ProtectSystem       = true
# We may want to change to on-failure, in which case service is restarted
# only if it did not exit cleanly previously.
Restart             = always
RestartSec          = 2
StandardOutput      = journal
StandardError       = journal
WorkingDirectory    = /run/racktop

[Install]
WantedBy = multi-user.target
ssh.config
content_copyfile_download
# /root/.ssh/config

Host *
    IdentityFile /etc/ssh/root/id_rsa
witness-notes.md
content_copyfile_download
  • Rendered
  • Source

Basic Steps to Complete on Witness Node

  • Generate self-signed SSL certificate for the witness.
> openssl genrsa -des3 -passout pass:x -out witness.pass.key 2048
...
> openssl rsa -passin pass:x -in witness.pass.key -out server.key

> rm witness.pass.key
> openssl req -new -key witness.key -out witness.csr
...
openssl x509 -req -sha256 -in witness.csr -signkey witness.key -out witness.pem

Prereqs

  • Make sure that private key for root exists in /etc/ssh/root/id_rsa, create it or copy from /root/.ssh.
  • Add peer ssh public keys to authorized_keys file on witness. Public keys should go into /root/.ssh/authorized_keys.
  • On each cluster peer add witness' ssh key to /etc/ssh/root/authorized_keys, to make sure each node can connect to witness.
  • Validate that each node can ssh into witness and execute commands on witness without password prompt.
  • Validate that witness can ssh into both nodes of the cluster.
  • On witness install ipmitool package so as to allow for ipmi control of the nodes in the cluster via IPMI.
  • Update /etc/hosts file with one entry for each node and witness. Using node1, node2 and witness as a standard seems like a good approach.
  • Copy startup script to /usr/racktop/sbin/hiavd-systemd.sh before installing with systemctl.
  • Copy hiavd.service file to /etc/systemd/system/hiavd.service before installing with systemctl.

Installation of Service

Use systemctl to enable hiavd.

# systemctl enable hiavd
Created symlink from /etc/systemd/system/multi-user.target.wants/hiavd.service to /etc/systemd/system/hiavd.service.

Start hiavd from systemd and observe its state. It should remain online, otherwise if service crashes it will be restarted, but pid will change, and is a good clue that service is crashing.

# systemctl status hiavd
● hiavd.service - RackTop Systems High Availability Daemon (hiavd)
   Loaded: loaded (/etc/systemd/system/hiavd.service; enabled; vendor preset: disabled)
   Active: inactive (dead)

# systemctl start hiavd

# systemctl status hiavd
● hiavd.service - RackTop Systems High Availability Daemon (hiavd)
   Loaded: loaded (/etc/systemd/system/hiavd.service; enabled; vendor preset: disabled)
   Active: active (running) since Wed 2017-12-20 09:31:34 EST; 2s ago
 Main PID: 1406 (hiavd-systemd.s)
   CGroup: /system.slice/hiavd.service
           ├─1406 /bin/bash /usr/racktop/sbin/hiavd-systemd.sh start
           └─1407 /usr/racktop/sbin/hiavd -w /run/racktop -pid /run/racktop/hiavd.pid

Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Ente...r
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Ente...r
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Debug] Setu....
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Leav...r
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Ente...e
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Ente...r
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Debug] Init....
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Leav...r
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Trace] Leav...e
Dec 20 09:31:35 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:31:35 [Debug] Chan....
Hint: Some lines were ellipsized, use -l to show in full.

To undo this, perform steps in reverse order. Stop the service with systemctl and then disable.

# systemctl stop hiavd

# systemctl status hiavd
● hiavd.service - RackTop Systems High Availability Daemon (hiavd)
   Loaded: loaded (/etc/systemd/system/hiavd.service; enabled; vendor preset: disabled)
   Active: inactive (dead) since Wed 2017-12-20 09:34:05 EST; 2s ago
  Process: 1425 ExecStop=/usr/racktop/sbin/hiavd-systemd.sh stop (code=exited, status=0/SUCCESS)
  Process: 1406 ExecStart=/usr/racktop/sbin/hiavd-systemd.sh start (code=killed, signal=TERM)
 Main PID: 1406 (code=killed, signal=TERM)

Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Trace] Ente...y
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Trace] Leav...y
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Trace] Ente...e
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Trace] Leav...e
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Trace] Leav...a
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Debug] Remo....
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Info] Servi....
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]: 2017-12-20/09:34:05 [Debug] Chan....
Dec 20 09:34:05 witness1.racktoplabs.com systemd[1]: Stopped RackTop Systems High Availability D...).
Dec 20 09:34:05 witness1.racktoplabs.com hiavd-systemd.sh[1406]:
Hint: Some lines were ellipsized, use -l to show in full.

# systemctl disable hiavd
Removed symlink /etc/systemd/system/multi-user.target.wants/hiavd.service.