szaydel
2/13/2015 - 5:28 PM

[Dtrace snippets for ARC + Memory] Obsrerve ARC behavior, system memory usage, memory pressure, etc. #tags: arc, memory, memory pressure, av

[Dtrace snippets for ARC + Memory] Obsrerve ARC behavior, system memory usage, memory pressure, etc. #tags: arc, memory, memory pressure, availrmem, available memory, pages, page usage, available pages, system paging, paging behavior

dtrace -qn 'tick-1sec {
    printf("total=%lu freemem=%lu availrmem=%lu locked=%lu user_locked=%lu\n",
    (pgcnt_t)`total_pages, /* number of pages used by /proc */
    (pgcnt_t)`freemem,
    (pgcnt_t)`availrmem, 
    (pgcnt_t)`availrmem_initial - (pgcnt_t)`availrmem,
    (pgcnt_t)`pages_locked
    );
}'
dtrace -qn '
    BEGIN {
        state["new_state-mru"] = "=> MRU";
        state["new_state-mfu"] = "=> MFU"
    }
    ::zfs:arc_access*:new* {
        @[state[probename]] = count();
    }
    END {
        printa("%s\t%@d\n", @);
    }'
// Will print "Yes" if reclaim needed, otherwise "No".
dtrace -qn '::arc_reclaim_needed:return { 
    @[args[1] > 0 ? "Yes" : "No"] = count(); 
    }
tick-5sec {
    printf("%Y\t", walltimestamp); printa("%s %@d", @); printf("\n"); trunc(@);
    }'
// Watch evictions or recycle events of ARC buffers, grouped by type and quantized on size.
dtrace -qn 'tick-5sec {
    ts = walltimestamp;
}
fbt::arc_evict:entry {
    @[ts, args[0], arg3 == 0 ? "evict" : "recycle", arg4 == 0 ? "data" : "metadata"] = quantize(arg2);
}
tick-10sec {
    printa("%16Y %a %s of %s %@d\n", @); trunc(@);
}'
// CSV parsable output, with a 5 second granularity of total bytes in ARC buffer frees for that 5 second period.
dtrace -qn 'tick-5sec {
    ts = walltimestamp;
    }
    ::arc_buf_data_free:entry /ts/ {
    @b[ts] = sum(args[0]->b_hdr->b_size);
    }
    tick-10sec {
    printa("%Y,%@d\n", @b); 
    trunc(@b);
    ts = 0;
    }'
// Quantized distribution of sizes of buffers being freed. May be useful to know if system runs into some bottlenecks at particular periods.
dtrace -qn 'tick-1sec {
    ts = walltimestamp;
    }
    ::arc_buf_data_free:entry /ts/ {
    @b["bufsize", ts] = quantize(args[0]->b_hdr->b_size);
    }
    tick-5sec {
    printa("%8s %16Y %@d\n", @b); 
    trunc(@b);
    ts = 0;
    }'
// Get a per-second resolution of arc_available_memory, which when negative means reclaim of memory is necessary and could be indicative of periods of memory pressure.

dtrace -qn 'tick-1sec {
    ts = walltimestamp; 
    } 
    ::arc_available_memory:return /ts/ {
    @avail[ts, args[1] >> 20] = count();
    } 
    tick-5sec{
    printa("%Y,%d MB,%@d\n", @avail); trunc(@avail); ts = 0;
    }'
// How much memory is available for ARC to take. Output csv for easier post-processing.
dtrace -qn '
    BEGIN {
        flag=0;
        cnt=0;
        printf("timestamp,min,max,maxdelta\n");
    }
    ::arc_available_memory:return /flag == 0/ {
        mmin = args[1];
        mmax = args[1];
        flag = 1; /* Set initial value of mmin/mmax, reset flag every 10s. */
    }
    ::arc_available_memory:return /flag == 1/ {
        mmin = (args[1] < mmin) ? args[1] : mmin;
        mmax = (args[1] > mmax) ? args[1] : mmax;
        delta = mmax - mmin;
        ++cnt;
    } 
    tick-10sec /flag && cnt > 1/ {
        printf("%Y,%d,%d,%d\n",walltimestamp, mmin, mmax, delta);
        ok=0; cnt=0; mmin=0; mmax=0; delta=0; flag=0;
    }'
#!/usr/sbin/dtrace -qs

::arc_adjust:entry,::arc_kmem_reap_now:entry {
    self->x[curpsinfo->pr_psargs] = timestamp;
}

::arc_shrink:entry
{
    trace("arc_shrink entered");
}

::arc_kmem_reap_now:return,
::arc_adjust:return
/self->x[curpsinfo->pr_psargs] && ((timestamp - self->x[curpsinfo->pr_psargs]) / 1000000 > 0)/
{
    printf("%Y %d ms\n", walltimestamp,
        (timestamp - self->x[curpsinfo->pr_psargs]) / 1000000);
    self->x[curpsinfo->pr_psargs] = 0;
}
#!/usr/sbin/dtrace -qs

::arc_adjust:entry,::arc_kmem_reap_now:entry {
    self->x[curpsinfo->pr_psargs] = timestamp;
}

::arc_shrink:entry
{
    trace("arc_shrink entered");
}

::arc_kmem_reap_now:return,
::arc_adjust:return
/self->x[curpsinfo->pr_psargs] && ((timestamp - self->x[curpsinfo->pr_psargs]) / 1000000 > 0)/
{
    printf("%Y %d ms\n", walltimestamp,
        (timestamp - self->x[curpsinfo->pr_psargs]) / 1000000);
    self->x[curpsinfo->pr_psargs] = 0;
}