[Dtrace snippets for ARC + Memory] Obsrerve ARC behavior, system memory usage, memory pressure, etc. #tags: arc, memory, memory pressure, availrmem, available memory, pages, page usage, available pages, system paging, paging behavior
dtrace -qn 'tick-1sec {
printf("total=%lu freemem=%lu availrmem=%lu locked=%lu user_locked=%lu\n",
(pgcnt_t)`total_pages, /* number of pages used by /proc */
(pgcnt_t)`freemem,
(pgcnt_t)`availrmem,
(pgcnt_t)`availrmem_initial - (pgcnt_t)`availrmem,
(pgcnt_t)`pages_locked
);
}'
dtrace -qn '
BEGIN {
state["new_state-mru"] = "=> MRU";
state["new_state-mfu"] = "=> MFU"
}
::zfs:arc_access*:new* {
@[state[probename]] = count();
}
END {
printa("%s\t%@d\n", @);
}'
// Will print "Yes" if reclaim needed, otherwise "No".
dtrace -qn '::arc_reclaim_needed:return {
@[args[1] > 0 ? "Yes" : "No"] = count();
}
tick-5sec {
printf("%Y\t", walltimestamp); printa("%s %@d", @); printf("\n"); trunc(@);
}'
// Watch evictions or recycle events of ARC buffers, grouped by type and quantized on size.
dtrace -qn 'tick-5sec {
ts = walltimestamp;
}
fbt::arc_evict:entry {
@[ts, args[0], arg3 == 0 ? "evict" : "recycle", arg4 == 0 ? "data" : "metadata"] = quantize(arg2);
}
tick-10sec {
printa("%16Y %a %s of %s %@d\n", @); trunc(@);
}'
// CSV parsable output, with a 5 second granularity of total bytes in ARC buffer frees for that 5 second period.
dtrace -qn 'tick-5sec {
ts = walltimestamp;
}
::arc_buf_data_free:entry /ts/ {
@b[ts] = sum(args[0]->b_hdr->b_size);
}
tick-10sec {
printa("%Y,%@d\n", @b);
trunc(@b);
ts = 0;
}'
// Quantized distribution of sizes of buffers being freed. May be useful to know if system runs into some bottlenecks at particular periods.
dtrace -qn 'tick-1sec {
ts = walltimestamp;
}
::arc_buf_data_free:entry /ts/ {
@b["bufsize", ts] = quantize(args[0]->b_hdr->b_size);
}
tick-5sec {
printa("%8s %16Y %@d\n", @b);
trunc(@b);
ts = 0;
}'
// Get a per-second resolution of arc_available_memory, which when negative means reclaim of memory is necessary and could be indicative of periods of memory pressure.
dtrace -qn 'tick-1sec {
ts = walltimestamp;
}
::arc_available_memory:return /ts/ {
@avail[ts, args[1] >> 20] = count();
}
tick-5sec{
printa("%Y,%d MB,%@d\n", @avail); trunc(@avail); ts = 0;
}'
// How much memory is available for ARC to take. Output csv for easier post-processing.
dtrace -qn '
BEGIN {
flag=0;
cnt=0;
printf("timestamp,min,max,maxdelta\n");
}
::arc_available_memory:return /flag == 0/ {
mmin = args[1];
mmax = args[1];
flag = 1; /* Set initial value of mmin/mmax, reset flag every 10s. */
}
::arc_available_memory:return /flag == 1/ {
mmin = (args[1] < mmin) ? args[1] : mmin;
mmax = (args[1] > mmax) ? args[1] : mmax;
delta = mmax - mmin;
++cnt;
}
tick-10sec /flag && cnt > 1/ {
printf("%Y,%d,%d,%d\n",walltimestamp, mmin, mmax, delta);
ok=0; cnt=0; mmin=0; mmax=0; delta=0; flag=0;
}'
#!/usr/sbin/dtrace -qs
::arc_adjust:entry,::arc_kmem_reap_now:entry {
self->x[curpsinfo->pr_psargs] = timestamp;
}
::arc_shrink:entry
{
trace("arc_shrink entered");
}
::arc_kmem_reap_now:return,
::arc_adjust:return
/self->x[curpsinfo->pr_psargs] && ((timestamp - self->x[curpsinfo->pr_psargs]) / 1000000 > 0)/
{
printf("%Y %d ms\n", walltimestamp,
(timestamp - self->x[curpsinfo->pr_psargs]) / 1000000);
self->x[curpsinfo->pr_psargs] = 0;
}
#!/usr/sbin/dtrace -qs
::arc_adjust:entry,::arc_kmem_reap_now:entry {
self->x[curpsinfo->pr_psargs] = timestamp;
}
::arc_shrink:entry
{
trace("arc_shrink entered");
}
::arc_kmem_reap_now:return,
::arc_adjust:return
/self->x[curpsinfo->pr_psargs] && ((timestamp - self->x[curpsinfo->pr_psargs]) / 1000000 > 0)/
{
printf("%Y %d ms\n", walltimestamp,
(timestamp - self->x[curpsinfo->pr_psargs]) / 1000000);
self->x[curpsinfo->pr_psargs] = 0;
}