szaydel
1/25/2018 - 4:11 PM

Dtrace ARC Snippets for memory shortfall and reclaim observations

#!/usr/sbin/dtrace -qCs
/*
 * Measure how much time is spent in arc_kmem_reap_now function.
 * There are known issues with spending a long time in this call.
 * We want to know how much time was spent and how much was freed.
 */
BEGIN {
  printf("timestamp,callDuration,pagesFree,pagesReclaimed\n");
}
::arc_kmem_reap_now:entry {
  self->in=timestamp;
  this->free1 = `freemem;
}
::arc_kmem_reap_now:return /self->in/ {
  this->free2 = `freemem;
  /* How much we freed since entry into this function, could be negative! */
  this->freed = this->free2 - this->free1;
  self->in = 0;
  printf("%ld,%u,%lu,%ld\n", walltimestamp,
          (timestamp-self->in)/1000, this->free1, (long)this->freed);
}
#!/usr/sbin/dtrace -qCs
/*
  * Snippet collects minimum and maximum available bytes to ARC.
  * Measurement is coming from a periodic call to  arc_available_memory,
  * which returns a signed value, where anything below zero signals
  * memory shortfall, and will result in reclaim activity.
  */
inline const char MIN = 0;
inline const char MAX = 1;
int shortfall; int x; char reclaim; long l[char];

BEGIN { printf("timestamp,minAvail,maxAvail,reclaim,shortCnt\n"); }
::arc_available_memory:return /!x/ {
  l[MIN] = l[MAX] = args[1] ; x++ ;
  reclaim = 0 ; shortfall = 0 ;
}
::arc_available_memory:return /x/ { /* Record minimum and maximum values */
  l[MIN] = args[1] < l[MIN] ? args[1] : l[MIN] ;
  l[MAX] = args[1] > l[MAX] ? args[1] : l[MAX] ;
  reclaim = reclaim == 1 ? 1 : l[MIN] < 0 ? 1 : 0 ; /* reclaim needed? */
  shortfall += args[1] < 0 ? 1 : 0 ;
}
tick-5sec {
  printf("%ld,%ld,%ld,%d,%d\n", 
          walltimestamp, l[MIN], l[MAX], reclaim, shortfall) ;
  x = 0 ;
}
dtrace -qn '
  /*
   * Snippet collects minimum and maximum available bytes to ARC.
   * Measurement is coming from a periodic call to  arc_available_memory,
   * which returns a signed value, where anything below zero signals
   * memory shortfall, and will result in reclaim activity.
   */
  inline const char MIN = 0;
  inline const char MAX = 1;
  int shortfall; int x; char reclaim; long l[char];

  BEGIN { printf("timestamp,minAvail,maxAvail,reclaim\n"); }
  ::arc_available_memory:return /!x/ {
    l[MIN] = l[MAX] = args[1] ; x++ ;
    reclaim = 0 ; shortfall = 0 ;
  }
  ::arc_available_memory:return /x/ { /* Record minimum and maximum values */
    l[MIN] = args[1] < l[MIN] ? args[1] : l[MIN] ;
    l[MAX] = args[1] > l[MAX] ? args[1] : l[MAX] ;
    reclaim = reclaim == 1 ? 1 : l[MIN] < 0 ? 1 : 0 ; /* reclaim needed? */
    shortfall += args[1] < 0 ? 1 : 0 ;
  }
  tick-5sec {
    printf("%ld,%ld,%ld,%d,%d\n", 
          walltimestamp, l[MIN], l[MAX], reclaim, shortfall) ;
    x = 0 ;
  }'