szaydel
8/24/2016 - 7:13 PM
Panic analysis
Panic analysis
If we look at the process and the associated thread, we find its stack, which is below. This address can be seen throughout the stack below, and ties buffer to the thread, in kernel, from which we trace back to program that correlates with the given kernel thread.
> fffffea3f5455140::print struct buf b_proc| ::threadlist
ADDR PROC LWP CMD/LWPID
fffffea35a332010 fffffea3599ab140 0 0xfffffea31d21c9c0()
> fffffea3f5455140::print struct buf b_proc|::ps
S PID PPID PGID SID UID FLAGS ADDR NAME
R 15736 2067 10 10 0 0x4a004000 fffffea35a332010 rtdmgr
This is the stack for that process in kernel. Now, what we don't know if we are doing something wrong, or if the problem is with how it is handled in the kernel.
> fffffea3599ab140::findstack -v
stack pointer for thread fffffea3599ab140: fffffe422822e8a0
[ fffffe422822e8a0 _resume_from_idle+0x112() ]
fffffe422822e8d0 swtch+0x12e()
fffffe422822e910 sema_p+0x1d1(fffffea3f5455200)
fffffe422822e950 biowait+0xa4(fffffea3f5455140)
fffffe422822ea20 default_physio+0x2a8(fffffffff795bc90, fffffea3f5455140, 5300000040, 40,
fffffffffbb04c40, fffffe422822ea90)
fffffe422822ea60 physio+0x11(fffffffff795bc90, fffffea3f5455140, 5300000040, 40, fffffffffbb04c40,
fffffe422822ea90)
fffffe422822eb20 scsi_uscsi_handle_cmd+0x27d(5300000040, 0, fffffea3d67df9b0, fffffffff795bc90, 0,
fffffea3d6cfee70)
fffffe422822ebb0 sd_ssc_send+0x10f(fffffea3d552f000, 7f9bb7c, 100083, 0, 0)
fffffe422822ec90 sdioctl+0x138f(5300000040, 4c9, 7f9bb7c, 100083, fffffea38379ba70,
fffffe422822ede0)
fffffe422822ecd0 cdev_ioctl+0x25(5300000040, 4c9, 7f9bb7c, 100083, fffffea38379ba70,
fffffe422822ede0)
fffffe422822ed20 spec_ioctl+0x4d(fffffea35a393640, 4c9, 7f9bb7c, 100083, fffffea38379ba70,
fffffe422822ede0, 0)
fffffe422822edb0 fop_ioctl+0x3b(fffffea35a393640, 4c9, 7f9bb7c, 100083, fffffea38379ba70,
fffffe422822ede0, 0)
fffffe422822eec0 ioctl+0x97(4, 4c9, 7f9bb7c)
fffffe422822ef10 _sys_sysenter_post_swapgs+0x149()
> fffffea3f5455140::print struct buf b_proc|::ps
S PID PPID PGID SID UID FLAGS ADDR NAME
R 15736 2067 10 10 0 0x4a004000 fffffea35a332010 rtdmgr
This is the uscsi_cmd struct, if we believe scsi_uscsi_handle_cmd:
> fffffea3d67df9b0::print -ta struct uscsi_cmd
fffffea3d67df9b0 struct uscsi_cmd {
fffffea3d67df9b0 int uscsi_flags = 0x1000f
fffffea3d67df9b4 short uscsi_status = 0
fffffea3d67df9b6 short uscsi_timeout = 0x5
fffffea3d67df9b8 caddr_t uscsi_cdb = 0xfffffea3d5a313c0
fffffea3d67df9c0 caddr_t uscsi_bufaddr = 0x7f9bc20
fffffea3d67df9c8 size_t uscsi_buflen = 0x400
fffffea3d67df9d0 size_t uscsi_resid = 0
fffffea3d67df9d8 uchar_t uscsi_cdblen = 0xa
fffffea3d67df9d9 uchar_t uscsi_rqlen = 0xfc
fffffea3d67df9da uchar_t uscsi_rqstatus = 0
fffffea3d67df9db uchar_t uscsi_rqresid = 0xfc
fffffea3d67df9e0 caddr_t uscsi_rqbuf = 0xfffffea3d5e4b040
fffffea3d67df9e8 ulong_t uscsi_path_instance = 0
And this is the CDB according to uscsi_cdb, which matches exactly with what you see lower, which was taken from the address pointed by the stack captured at the time of panic.
> fffffea3d67df9b0::print struct uscsi_cmd uscsi_cdb|::array uchar_t 0xa|::print uchar_t
0x4d
0
0x40
0
0
0
0
0x4
0
0
Length of CDB issued:
> fffffea3820e5678::print struct scsi_pkt pkt_cdblen
pkt_cdblen = 0xa
> fffffea3820e5678::print struct scsi_pkt pkt_cdbp|::array uchar_t 0xa | ::print uchar_t
0x4d
0
0x40
0
0
0
0
0x4
0
0
According to this call on the stack sd_lun is fffffea3574da9c0
fffffe42285e0970 sd_return_command+0xfe(fffffea3574da9c0, fffffea3f5455140)
This is the state of the drive at that time:
> fffffea3574da9c0::sd_state
un: fffffea3574da9c0
--------------
{
un_sd = 0xfffffea31c12b6d8
un_rqs_bp = 0xfffffea357534100
un_rqs_pktp = 0xfffffea35772fdd8
un_sense_isbusy = 0
un_buf_chain_type = 0x1
un_uscsi_chain_type = 0x8
un_direct_chain_type = 0x8
un_priority_chain_type = 0x9
un_waitq_headp = 0
un_waitq_tailp = 0
un_retry_bp = 0
un_retry_statp = 0
un_xbuf_attr = 0xfffffea327099700
un_sys_blocksize = 0x200
un_tgt_blocksize = 0x200
un_phy_blocksize = 0x200
un_blockcount = 0x7bffff
un_ctype = 0x2
un_node_type = 0xfffffffff7971497 "ddi_block:channel"
un_interconnect_type = 0x4
un_notready_retry_count = 0x2
un_busy_retry_count = 0x5
un_retry_count = 0x5
un_victim_retry_count = 0xa
un_reset_retry_count = 0x2
un_reserve_release_time = 0x5
un_reservation_type = 0x1
un_max_xfer_size = 0x100000
un_partial_dma_supported = 0x1
un_buf_breakup_supported = 0
un_mincdb = 0
un_maxcdb = 0x3
un_max_hba_cdb = 0x10
un_status_len = 0x20
un_pkt_flags = 0x40000
un_cmd_timeout = 0x3c
un_uscsi_timeout = 0x3c
un_busy_timeout = 0x1f4
un_state = 0
un_last_state = 0
un_last_pkt_reason = 0
un_tagflags = 0x4000
un_resvd_status = 0
un_detach_count = 0
un_layer_count = 0
un_opens_in_progress = 0
un_semoclose = {
_opaque = [ 0, 1 ]
}
un_ncmds_in_driver = 0x2
un_ncmds_in_transport = 0
un_throttle = 0x20
un_saved_throttle = 0x20
un_busy_throttle = 0
un_min_throttle = 0x8
un_reset_throttle_timeid = 0
un_mhd_token = 0
un_resvd_timeid = 0
un_insert_event = 0
un_insert_cb_id = 0
un_remove_event = 0
un_remove_cb_id = 0
un_start_stop_cycle_page = 0
un_dcvb_timeid = 0
un_ocmap = {
chkd = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, ... ]
rinfo = {
lyr_open = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
, 0, 0, 0, 0, 0, 0, ... ]
reg_open = [ 0x1, 0, 0x1, 0 ]
}
}
un_pstats = [ 0xfffffea357ab6330, 0, 0, 0, 0, 0, 0, 0xfffffea357ab6490, 0xfffffea357ab65f0, 0, 0,
0, 0, 0, 0, 0, 0xfffffea357ab6750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... ]
un_stats = 0xfffffea357781440
un_errstats = 0xfffffea3577815a0
un_exclopen = 0
un_devid = 0xfffffea3577466c0
un_vpd_page_mask = 0x13
un_f_arq_enabled = 0x1
un_f_blockcount_is_valid = 0x1
un_f_tgt_blocksize_is_valid = 0x1
un_f_allow_bus_device_reset = 0x1
un_f_is_fibre = 0
un_f_sync_cache_supported = 0x1
un_f_format_in_progress = 0
un_f_opt_queueing = 0
un_f_opt_fab_devid = 0
un_f_opt_disable_cache = 0
un_f_cfg_is_atapi = 0
un_f_write_cache_enabled = 0
un_f_cfg_playmsf_bcd = 0
un_f_cfg_readsub_bcd = 0
un_f_cfg_read_toc_trk_bcd = 0
un_f_cfg_read_toc_addr_bcd = 0
un_f_cfg_no_read_header = 0
un_f_cfg_read_cd_xd4 = 0
un_f_mmc_cap = 0
un_f_mmc_writable_media = 0
un_f_dvdram_writable_device = 0
un_f_cfg_cdda = 0
un_f_cfg_tur_check = 0
un_f_use_adaptive_throttle = 0
un_f_pm_is_enabled = 0
un_f_watcht_stopped = 0
un_f_pkstats_enabled = 0x1
un_f_disksort_disabled = 0x1
un_f_lun_reset_enabled = 0
un_f_doorlock_supported = 0
un_f_start_stop_supported = 0
un_f_reserved1 = 0
un_f_mboot_supported = 0
un_f_is_hotpluggable = 0
un_f_has_removable_media = 0
un_f_non_devbsize_supported = 0
un_f_devid_supported = 0x1
un_f_eject_media_supported = 0
un_f_chk_wp_open = 0
un_f_descr_format_supported = 0x1
un_f_check_start_stop = 0
un_f_monitor_media_state = 0
un_f_attach_spinup = 0x1
un_f_log_sense_supported = 0x1
un_f_pm_supported = 0
un_f_cfg_is_lsi = 0
un_f_wcc_inprog = 0
un_f_ejecting = 0
un_f_suppress_cache_flush = 0
un_f_sync_nv_supported = 0
un_f_sync_cache_required = 0x1
un_f_devid_transport_defined = 0
un_f_rmw_type = 0
un_f_power_condition_disabled = 0
un_f_power_condition_supported = 0
un_f_pm_log_sense_smart = 0
un_f_is_solid_state = 0
un_f_mmc_gesn_polling = 0x1
un_f_enable_rmw = 0
un_f_expnevent = 0
un_f_reserved = 0
un_additional_codes = 0
un_pm_mutex = {
_opaque = [ 0 ]
}
un_pm_count = 0
un_pm_timeid = 0
un_pm_busy = 0
un_pm_busy_cv = {
_opaque = 0
}
un_power_level = 0x1
un_save_state = 0
un_suspend_cv = {
_opaque = 0
}
un_disk_busy_cv = {
_opaque = 0
}
un_state_cv = {
_opaque = 0
}
un_mediastate = 2 (DKIO_INSERTED)
un_specified_mediastate = 0 (DKIO_NONE)
un_swr_token = 0
un_wm_cache = 0
un_rmw_count = 0
un_wm = 0
un_rmw_incre_count = 0
un_rmw_msg_timeid = 0
un_startstop_timeid = 0
un_direct_priority_timeid = 0
un_tran_fatal_count = 0
un_retry_timeid = 0
un_pm_idle_time = 0x313f2ce59f1c
un_pm_idle_timeid = 0
un_sonoma_failure_count = 0
un_failfast_bp = 0
un_failfast_headp = 0
un_failfast_tailp = 0
un_failfast_state = 0
un_in_callback = 0x1
un_wcc_cv = {
_opaque = 0
}
un_cmlbhandle = 0xfffffea35772d000
un_fm_private = 0xfffffea31d4d9000
}
---
XBUF ATTR:
----------
0xfffffea327099700: mutex allocsize pending
0 128 0
0xfffffea327099714: active_limit active_count active_lowater
200 0 100
0xfffffea327099720: headp tailp
0 fffffea3f53f1e80
0xfffffea327099730: reserve_mutex reserve_limit reserve_count reserve_headp
0 10 10 fffffea36ccd1600
0xfffffea327099758: timeid tq
0 fffffea3575299f8
---
XBUF Q:
-------
xbuf Q head: 0
---------------------------
Processed 0 XBUF Q entries
---------------------------
UN WAIT Q:
----------
UN wait Q head: 0
------------------------------
Processed 0 UN WAIT Q entries
------------------------------
This is the packet with which we are
> fffffea3820e5678::print -ta struct scsi_pkt
fffffea3820e5678 struct scsi_pkt {
fffffea3820e5678 opaque_t pkt_ha_private = 0xfffffea3820e5758
fffffea3820e5680 struct scsi_address pkt_address = {
fffffea3820e5680 struct scsi_hba_tran *a_hba_tran = 0xfffffea35752b600
fffffea3820e5688 union a = {
fffffea3820e5688 struct spi = {
fffffea3820e5688 ushort_t a_target = 0
fffffea3820e568a uchar_t a_lun = 0
fffffea3820e568b uchar_t _a_sublun = 0
}
fffffea3820e5688 struct scsi_device *a_sd = 0
}
}
fffffea3820e5690 opaque_t pkt_private = 0xfffffea3f5455140
fffffea3820e5698 void (*)() pkt_comp = sdintr
fffffea3820e56a0 uint_t pkt_flags = 0x20074000
fffffea3820e56a4 int pkt_time = 0x5
fffffea3820e56a8 uchar_t *pkt_scbp = 0xfffffea3f4fc4180
fffffea3820e56b0 uchar_t *pkt_cdbp = 0xfffffea3820e5758
fffffea3820e56b8 ssize_t pkt_resid = 0
fffffea3820e56c0 uint_t pkt_state = 0x1f
fffffea3820e56c4 uint_t pkt_statistics = 0
fffffea3820e56c8 uchar_t pkt_reason = 0
fffffea3820e56cc uint_t pkt_cdblen = 0xa
fffffea3820e56d0 uint_t pkt_tgtlen = 0
fffffea3820e56d4 uint_t pkt_scblen = 0x108
fffffea3820e56d8 ddi_dma_handle_t pkt_handle = 0xfffffea3dffc8ac0
fffffea3820e56e0 uint_t pkt_numcookies = 0x2
fffffea3820e56e8 off_t pkt_dma_offset = 0
fffffea3820e56f0 size_t pkt_dma_len = 0x400
fffffea3820e56f8 uint_t pkt_dma_flags = 0x2
fffffea3820e5700 ddi_dma_cookie_t *pkt_cookies = 0xfffffea3dffc8cc8
fffffea3820e5708 int pkt_path_instance = 0
fffffea3820e5710 void *pkt_stmp = 0
}