Fabric everywhere
import StringIO
from fabric.api import *
env.hosts = [""]
krbconf = StringIO.StringIO("""
[libdefaults]
default_realm = CERN.CH
ticket_lifetime = 24h
renew_lifetime = 7d
forwardable = true
[realms]
CERN.CH = {
default_domain = cern.ch
kdc = cerndc.cern.ch
}
[domain_realm]
cern.ch = CERN.CH
.cern.ch = CERN.CH """)
def setup_afs(user="kostams"):
packages = [
"krb5-workstation", "krb5-libs", "krb5-auth-dialog",
"openafs-client", "openafs-server", "openafs-krb5",
"kernel-module-openafs", "compat-readline5.x86_64",
"openssl098e.x86_64", "fuse-libs.x86_64"]
run("yum -y install " + " ".join(packages))
put(krbconf, "/etc/krb5.conf")
run("kinit {0}@CERN.CH".format(user))
with settings(warn_only=True):
run("service afs start")
run("aklog")
cvmfs = StringIO.StringIO("""
CVMFS_HTTP_PROXY=\"http://ca-proxy.cern.ch:3128;http://ca-proxy1.cern.ch:3128|http://ca-proxy2.cern.ch:3128|http://ca-proxy3.cern.ch:3128|http://ca-proxy4.cern.ch:3128|http://ca-proxy5.cern.ch:3128\"
CVMFS_REPOSITORIES=ams
""")
def setup_cvmfs():
with cd("/etc/yum.repos.d/"):
run("wget http://cvmrepo.web.cern.ch/cvmrepo/yum/cernvm.repo")
with cd("/etc/pki/rpm-gpg/"):
run("wget http://cvmrepo.web.cern.ch/cvmrepo/yum/RPM-GPG-KEY-CernVM")
run("yum -y install cvmfs.x86_64")
put(cvmfs, "/etc/cvmfs/default.local")
put(StringIO.StringIO(
'CVMFS_SERVER_URL="http://cvmfs-stratum-one.cern.ch/opt/@org@"'
), "/etc/cvmfs/config.d/ams.cern.ch.local"
)
run("service autofs start")
run("cvmfs_config setup")
run("cvmfs_config probe")
def mount_all(user="kostams"):
afshome = '/afs/cern.ch/user/{0}/{1}'.format(user[0],user)
run("kinit {0}@CERN.CH".format(user))
run("aklog")
run("GROUP=va /afs/cern.ch/project/eos/installation/0.3.15/bin/eos.select -b fuse mount {0}/eos".format(afshome))
run("if ! grep -qs '/dev/vda3' /proc/mounts; then mount /dev/vda3 /data; fi")
def create_partition():
fdisk = run("fdisk -l -u /dev/vda")
N, last, ptype = None, None, None
for l in fdisk.split('\n'):
if not l.startswith("/dev/vda"):
continue
pinfo = l.split()
if pinfo[1] == '*': del pinfo[1]
N, last, ptype = int(pinfo[0][-1]), int(pinfo[2]), pinfo[4]
print "{0}: Found partition '{1}'".format(N,pinfo[0])
print " -- last block {}".format(last)
print " -- type {}".format(ptype)
with settings(warn_only=True):
run("echo -e 'u\nn\np\n{0}\n{1}\n\nt\n{0}\n{2}\nw\n' | fdisk /dev/vda".format(N+1,last+1,ptype))
reboot()
run("mkfs.ext4 /dev/vda{}".format(N+1))
import StringIO
from fabric.api import *
BASEDIR = "VirtualEnv"
DISTDIR = BASEDIR + "/distribs"
INSTALLDIR = BASEDIR + "/install"
env.hosts = [
# "root@kanishev-ams-vm",
"root@kanishev-ams-vm1",
# "root@kanishev-ams-vm2",
# "root@kanishev-ams-vm3",
]
envsh = StringIO.StringIO("""
MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $MY_PATH/install/bin/thisroot.sh
source $MY_PATH/ipython/bin/activate
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MY_PATH/install/lib
""")
def yum():
packages = [
"openssl-devel", "xorg-x11-server-devel", "gfortran",
"gcc-c++", "gcc", "binutils", "libX11-devel",
"libXpm-devel", "libXft-devel", "libXext-devel"]
run("yum -y install " + " ".join(packages))
def get_sources():
packages = [
"https://sqlite.org/2014/sqlite-autoconf-3080403.tar.gz",
"http://download.zeromq.org/zeromq-4.0.3.tar.gz",
"https://www.python.org/ftp/python/2.7.6/Python-2.7.6.tgz",
"https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.tar.gz",
"ftp://root.cern.ch/root/root_v5.34.18.source.tar.gz"
]
run("mkdir -p " + DISTDIR)
with cd(DISTDIR):
for p in packages:
run("wget " + p)
for p in packages:
run("tar xvvf " + p.split("/")[-1])
def compile():
run("mkdir -p " + INSTALLDIR)
with cd(DISTDIR + "/sqlite-autoconf-*"):
run("./configure --prefix=$HOME/" + INSTALLDIR)
run("make && make install")
with cd(DISTDIR + "/zeromq-*"):
run("./configure --prefix=$HOME/" + INSTALLDIR)
run("make && make install")
with cd(DISTDIR + "/Python-2.7.6"):
run('./configure CXXFLAGS="-I$HOME/{0}/include" --prefix=$HOME/{0} --enable-shared --enable-unicode=ucs4'.format(INSTALLDIR))
run("make && make install")
with cd("$HOME/{0}/lib".format(INSTALLDIR)):
run("ln -s libpython2.7.so libpython.so")
with cd("$HOME/{0}/root".format(DISTDIR)):
version = "4.1.1"
instdir = "$HOME/{0}/opt".format(INSTALLDIR)
run("mkdir -p " + instdir)
run("./build/unix/installXrootd.sh -v {0} $HOME/{1}/opt".format(version,INSTALLDIR)
confcommand = [
"./configure",
"--with-python-incdir=$HOME/{0}/include/python2.7/".format(INSTALLDIR),
"--with-python-libdir=$HOME/{0}/lib".format(INSTALLDIR),
"--prefix=$HOME/{0}".format(INSTALLDIR),
"--etcdir=$HOME/{0}/etc".format(INSTALLDIR),
"--with-xrootd-incdir=$HOME/{0}/opt/xrootd-{1}/include/xrootd".format(INSTALLDIR, version),
"--with-xrootd-libdir=$HOME/{0}/opt/xrootd-{1}/lib".format(INSTALLDIR, version) ]
run(" ".join(confcommand))
run("make && make install")
def install_numpy():
with cd(DISTDIR):
run("git clone git://github.com/xianyi/OpenBLAS")
with("cd OpenBLAS"):
run("make FC=gfortran")
run("make PREFIX=$HOME/{0} install".format(INSTALLDIR))
with cd(DISTDIR):
run("mkdir -p numpy")
run("pip install -d numpy numpy")
with cd("numpy"):
run("tar xvvf numpy-*.tar.gz")
with cd("numpy-*"):
put(StringIO.StringIO("""
[default]
library_dirs= $INSTALLDIR/lib/
[openblas]
libraries = openblas
library_dirs = $INSTALLDIR/lib
include_dirs = $INSTALLDIR/include"""), "site.cfg")
run("python setup.py install")
run("pip install scipy")
def prepare_venv():
with cd("$HOME/{0}".format(BASEDIR)):
run("cp $HOME/{0}/virtualenv-1.9/virtualenv.py .".format(DISTDIR))
run("LD_LIBRARY_PATH=$HOME/{0}/lib/ $HOME/{0}/bin/python virtualenv.py --python=$HOME/{0}/bin/python ipython".format(INSTALLDIR))
put(envsh, "env.sh")
run('source env.sh && pip install pyzmq --install-option="--zmq=$HOME/{0}"'.format(INSTALLDIR))
run('CPPFLAGS="-I$HOME/{0}/include" pip install pysqlite'.format(INSTALLDIR))
run('pip install ipython[all]')
export BASEDIR=$PWD
export INSTALLDIR=$PWD/install
export DISTDIR=$PWD/distrib
mkdir -p $INSTALLDIR
mkdir -p $DISTDIR
cd $DISTDIR
wget --no-check-certificate https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz
wget --no-check-certificate https://pypi.python.org/packages/source/v/virtualenv/virtualenv-12.0.tar.gz
tar xvvf virtualenv-*.tar.gz
tar xvvf Python-*.tgz
cd $DISTDIR/Python-*/
./configure CXXFLAGS="-I$INSTALLDIR/include" --prefix=$INSTALLDIR --enable-shared --enable-unicode=ucs4
make && make install
cd $DISTDIR/virtualenv-*/
LD_LIBRARY_PATH=$INSTALLDIR/lib/ $INSTALLDIR/bin/python setup.py install --prefix=$INSTALLDIR
cd $BASEDIR
LD_LIBRARY_PATH=$INSTALLDIR/lib/ $INSTALLDIR/bin/virtualenv --python=$INSTALLDIR/bin/python fabric
cat > env.sh << "EOF"
MY_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $MY_PATH/fabric/bin/activate
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MY_PATH/install/lib
EOF
. env.sh
pip install fabric