szaydel
7/27/2016 - 5:21 PM

Load Dynamics Results Data Parsing and Extraction Tools

Load Dynamics Results Data Parsing and Extraction Tools

#!/usr/bin/env python
'''
The MIT License (MIT)
=====================

Copyright (c) 2016 Sam Zaydel, RackTop Systems

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''

import sys

CSV_INPUT = "./ldnfslatency.csv"
COL_COUNT = 10
cmdMap = {
    "ALL": "all",
    "NFS_V3_READ": "read",
    "NFS_V3_WRITE": "write",
    "PMAP_NULL": "portmap.null",
    "PMAP_GETPORT": "portmap.getport",
    "MNT_V3_NULL": "mount.v3.null",
    "MNT_V3_MOUNT": "mount.v3.mount",
    "NFS_V3_NULL": "null",
    "NFS_V3_LOOKUP": "lookup",
    "NFS_V3_ACCESS": "access",
}

# Use this template to build a dict of dicts (bycmd) below,
# assuring all initial values are zeroed-out.
tpl = {
    "p_total_tm": 0,
    "p_tm_cnt": 0,
    "t_total_tm": 0,
    "t_tm_cnt": 0,
}

bycmd = {
    "ALL": tpl.copy(),
    "NFS_V3_READ": tpl.copy(),
    "NFS_V3_WRITE": tpl.copy(),
    "PMAP_NULL": tpl.copy(),
    "PMAP_GETPORT":tpl.copy(),
    "MNT_V3_NULL": tpl.copy(),
    "MNT_V3_MOUNT": tpl.copy(),
    "NFS_V3_NULL": tpl.copy(),
    "NFS_V3_LOOKUP": tpl.copy(),
    "NFS_V3_ACCESS": tpl.copy(),
    }

def main():
    # If there is another argument on the commad line, we should
    # treat it as file which we want to process, instead of default
    # value stored in CSV_INPUT.
    if len(sys.argv[:]) > 1:
        src = sys.argv[1]
    else:
        src = CSV_INPUT
    try:
        with open(src, 'rb') as data:
            # Print header, just once.
            print("stat.id,stat.time,rpc.cmd,latency,min.lat,max.lat")
            # Skip first line of file, it is a header.
            _ = data.readline()
            while True:
                line = data.readline().rstrip()
                if line == "":
                    break
                s = line.split(",")
                if len(s) == COL_COUNT:
                    stat_id = s[0]
                    stat_tm = s[1]
                    rpc_cmd  = s[3].strip(r'"')
                    total_tm = float(s[4])
                    tm_cnt = float(s[5])
                    tm_min = float(s[6])
                    tm_max = float(s[7])
                    bycmd[rpc_cmd]["t_total_tm"] = total_tm
                    bycmd[rpc_cmd]["t_tm_cnt"] = tm_cnt
                    total_tm_delta = \
                    bycmd[rpc_cmd]["t_total_tm"] - bycmd[rpc_cmd]["p_total_tm"]
                    tm_cnt_delta = \
                    bycmd[rpc_cmd]["t_tm_cnt"] - bycmd[rpc_cmd]["p_tm_cnt"]
                    # If either delta is zero, we don't want to do division.
                    if tm_cnt_delta == 0 or total_tm_delta == 0:
                        lat = total_tm_delta
                    else:
                        lat = total_tm_delta / tm_cnt_delta
                    # Map RAW command name to output name and convert floats
                    # to string values then print resulting line.
                    newline = ",".join(
                        [
                            stat_id, stat_tm, cmdMap[rpc_cmd],
                            str(lat), str(tm_min), str(tm_max)
                        ]
                    )
                    print(newline)  # Produce computed output
                    # Switch previous total and count for this total and count
                    # for next loop iteration.
                    bycmd[rpc_cmd]["p_total_tm"], bycmd[rpc_cmd]["p_tm_cnt"] = bycmd[rpc_cmd]["t_total_tm"], bycmd[rpc_cmd]["t_tm_cnt"]
                else:
                    continue
    except IOError as e:
        sys.stderr.write("Failed to process. File %s not found.\n" % e.filename)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())
'stats.id,stats.timestamp,stats.count,port.time,port.tx.packets,port.rx.packets,port.tx.bytes,port.rx.bytes,port.maxpackets,port.scenarios.max,load.type,load.desired,load.current,load.actions.attempts,load.actions.succeeds,load.actions.fails,load.actions.aborts,load.scenarios.attempts,load.scenarios.succeeds,load.scenarios.fails,load.scenarios.aborts,load.connections.attempts,load.connections.succeeds,load.connections.fails,load.connections.aborts,rpc.actions.attempts,rpc.actions.succeeds,rpc.actions.fails,rpc.actions.aborts,rpc.resultok.attempts,rpc.resultok.succeeds,rpc.resultok.fails,rpc.resultok.aborts,rpc.rx.bytes,rpc.tx.bytes,rpc.rx.packets,rpc.tx.packets,rpc.time.count,rpc.time.total,rpc.time.min,rpc.time.max,rpc.time.count.cti,rpc.time.total.cti,rpc.time.min.cti,rpc.time.max.cti,rpc.commands.nfs_v3.read.actions.attempts,rpc.commands.nfs_v3.read.actions.succeeds,rpc.commands.nfs_v3.read.actions.fails,rpc.commands.nfs_v3.read.actions.aborts,rpc.commands.nfs_v3.read.time.count,rpc.commands.nfs_v3.read.time.total,rpc.commands.nfs_v3.read.time.min,rpc.commands.nfs_v3.read.time.max,rpc.commands.nfs_v3.read.time.count.cti,rpc.commands.nfs_v3.read.time.total.cti,rpc.commands.nfs_v3.read.time.min.cti,rpc.commands.nfs_v3.read.time.max.cti,rpc.commands.nfs_v3.write.actions.attempts,rpc.commands.nfs_v3.write.actions.succeeds,rpc.commands.nfs_v3.write.actions.fails,rpc.commands.nfs_v3.write.actions.aborts,rpc.commands.nfs_v3.write.time.count,rpc.commands.nfs_v3.write.time.total,rpc.commands.nfs_v3.write.time.min,rpc.commands.nfs_v3.write.time.max,rpc.commands.nfs_v3.write.time.count.cti,rpc.commands.nfs_v3.write.time.total.cti,rpc.commands.nfs_v3.write.time.min.cti,rpc.commands.nfs_v3.write.time.max.cti'

Steps for extracting/preparing CSV-formatted Summary Statistics

This process will be similar for all protocols, not just NFS, but the CSV headers are assuming NFS in this case.

  1. After completion of a Test Suite from Results Explorer right-click on associated results directory and choose open.
  2. Rename summary files to p0.summary and p1.summary for easier command-line use.
  3. Run the following command, replacing pX with p0 or p1, signifying which port this information came from. This must be run separately for each port. Currently we are using a total of 2 ports.
swiftsumreader -e 'stats.id,stats.timestamp,stats.count,port.time,port.tx.packets,port.rx.packets,port.tx.bytes,port.rx.bytes,port.maxpackets,port.scenarios.max,load.type,load.desired,load.current,load.actions.attempts,load.actions.succeeds,load.actions.fails,load.actions.aborts,load.scenarios.attempts,load.scenarios.succeeds,load.scenarios.fails,load.scenarios.aborts,load.connections.attempts,load.connections.succeeds,load.connections.fails,load.connections.aborts,rpc.actions.attempts,rpc.actions.succeeds,rpc.actions.fails,rpc.actions.aborts,rpc.resultok.attempts,rpc.resultok.succeeds,rpc.resultok.fails,rpc.resultok.aborts,rpc.rx.bytes,rpc.tx.bytes,rpc.rx.packets,rpc.tx.packets,rpc.time.count,rpc.time.total,rpc.time.min,rpc.time.max,rpc.time.count.cti,rpc.time.total.cti,rpc.time.min.cti,rpc.time.max.cti,rpc.commands.nfs_v3.read.actions.attempts,rpc.commands.nfs_v3.read.actions.succeeds,rpc.commands.nfs_v3.read.actions.fails,rpc.commands.nfs_v3.read.actions.aborts,rpc.commands.nfs_v3.read.time.count,rpc.commands.nfs_v3.read.time.total,rpc.commands.nfs_v3.read.time.min,rpc.commands.nfs_v3.read.time.max,rpc.commands.nfs_v3.read.time.count.cti,rpc.commands.nfs_v3.read.time.total.cti,rpc.commands.nfs_v3.read.time.min.cti,rpc.commands.nfs_v3.read.time.max.cti,rpc.commands.nfs_v3.write.actions.attempts,rpc.commands.nfs_v3.write.actions.succeeds,rpc.commands.nfs_v3.write.actions.fails,rpc.commands.nfs_v3.write.actions.aborts,rpc.commands.nfs_v3.write.time.count,rpc.commands.nfs_v3.write.time.total,rpc.commands.nfs_v3.write.time.min,rpc.commands.nfs_v3.write.time.max,rpc.commands.nfs_v3.write.time.count.cti,rpc.commands.nfs_v3.write.time.total.cti,rpc.commands.nfs_v3.write.time.min.cti,rpc.commands.nfs_v3.write.time.max.cti' -o pXnfsv3stat-raw.csv .\pX.summary
  1. Append information about second port to information extracted about first port using something along the lines of awk '{ if (NR>1) {print $0} }' pXnfsv3stat-raw.csv >> combined.csv.
  2. Sort resulting file using first two fields which should create proper temporal ordering. csvsort -c 'stats.id,stats.timestamp' ./combined.csv > ./ld-nfsv3-seq-64k-128conn.csv