Project

General

Profile

Actions
Copy link

Bug #5572

closed

segfault with osmo-BSC in osmo_mgcpc_ep_ci_request (rare)

Added by keith almost 2 years ago. Updated over 1 year ago.

Status:
Resolved
Priority:
High
Assignee:
dexter
Category:
RTP/Media
Target version:
-
Start date:
05/26/2022
Due date:
% Done:

100%

Spec Reference:

Description

The line that crashed was added in https://cgit.osmocom.org/osmo-mgw/commit/?id=3ff71284fa90e5c26963db860590054f41169970

We can run for a few days before hitting this.

I don't have much relevant lead-up log captured at this time, only a backtrace.
Last Log line on the console is:

DMSC ERROR osmo_bsc_bssap.c:1284 SUBSCR_CONN(msc0-conn43820_subscr-IMSI-334020218960160-TMSI-0x6919a240)[0x555555aded00]{WAIT_CLEAR_CMD}: Event MT_DTAP not permitted  


Program received signal SIGSEGV, Segmentation fault.                                                                                                                             
0x00007ffff79ebd6d in osmo_mgcpc_ep_ci_request (ci=0x555555b2d570, verb=MGCP_VERB_DLCX, verb_info=0x0, notify=0x0, event_success=0, event_failure=0, notify_data=0x0)            
    at mgcp_client_endpoint_fsm.c:665                                                                                                                                            
665             LOG_CI_VERB(ci, LOGL_DEBUG, "notify=%s\n", osmo_fsm_inst_name(ci->notify.fi));                                                                                   
(gdb) bt
#0  0x00007ffff79ebd6d in osmo_mgcpc_ep_ci_request (ci=0x555555b2d570, verb=MGCP_VERB_DLCX, verb_info=0x0, notify=0x0, event_success=0, event_failure=0, notify_data=0x0)
    at mgcp_client_endpoint_fsm.c:665
#1  0x000055555559d52d in osmo_mgcpc_ep_ci_dlcx (ci=0x555555b2d570) at /usr/local/include/osmocom/mgcp_client/mgcp_client_endpoint_fsm.h:42
#2  0x000055555559d802 in assignment_reset (conn=0x555555b367c0) at assignment_fsm.c:134
#3  0x00005555555c84ad in gscon_release_lchans (conn=0x555555b367c0, do_rr_release=true, cause_rr=GSM48_RR_CAUSE_NORMAL) at bsc_subscr_conn_fsm.c:260
#4  0x00005555555c824e in gscon_fsm_wait_sccp_rlsd_onenter (fi=0x555555aded00, prev_state=6) at bsc_subscr_conn_fsm.c:215
#5  0x00007ffff7ad09d4 in state_chg (fi=0x555555aded00, new_state=7, keep_timer=false, timeout_ms=60000, T=-4, file=0x5555556b8876 "bsc_subscr_conn_fsm.c", line=971)
    at fsm.c:694
#6  0x00007ffff7ad0a37 in _osmo_fsm_inst_state_chg (fi=0x555555aded00, new_state=7, timeout_secs=60, T=-4, file=0x5555556b8876 "bsc_subscr_conn_fsm.c", line=971) at fsm.c:743
#7  0x00007ffff7aebe4b in _osmo_tdef_fsm_inst_state_chg (fi=0x555555aded00, state=7, timeouts_array=0x55555571d300 <conn_fsm_timeouts>,
    tdefs=0x5555557281a0 <gsm_network_T_defs>, default_timeout=-1, file=0x5555556b8876 "bsc_subscr_conn_fsm.c", line=971) at tdef.c:357
#8  0x00005555555cc763 in gscon_fsm_allstate (fi=0x555555aded00, event=4, data=0x7fffffffcd1c) at bsc_subscr_conn_fsm.c:971
#9  0x00007ffff7ad133e in _osmo_fsm_inst_dispatch (fi=0x555555aded00, event=4, data=0x7fffffffcd1c, file=0x5555556e8c96 "osmo_bsc_bssap.c", line=438) at fsm.c:860
#10 0x000055555566f32b in bssmap_handle_clear_cmd (conn=0x555555b367c0, msg=0x555555b19d00, length=4) at osmo_bsc_bssap.c:438
#11 0x0000555555673a0e in bssmap_rcvmsg_dt1 (conn=0x555555b367c0, msg=0x555555b19d00, length=4) at osmo_bsc_bssap.c:1172
#12 0x0000555555674997 in bsc_handle_dt (conn=0x555555b367c0, msg=0x555555b19d00, len=6) at osmo_bsc_bssap.c:1360
#13 0x000055555567fb3a in handle_data_from_msc (conn=0x555555b367c0, msg=0x555555b19d00) at osmo_bsc_sigtran.c:141
#14 0x0000555555680598 in sccp_sap_up (oph=0x555555b19d88, _scu=0x555555a61ca0) at osmo_bsc_sigtran.c:256
#15 0x00007ffff7a1acd4 in sccp_user_prim_up (scu=0x555555a61ca0, prim=0x555555b19d88) at sccp_user.c:177
#16 0x00007ffff7a17da2 in scu_gen_encode_and_send (conn=0x555555b3e690, event=11, xua=0x555555b189f0, primitive=1, operation=PRIM_OP_INDICATION) at sccp_scoc.c:805
#17 0x00007ffff7a188ee in scoc_fsm_active (fi=0x555555aaf730, event=11, data=0x555555b189f0) at sccp_scoc.c:1124                                                                
#18 0x00007ffff7ad162d in _osmo_fsm_inst_dispatch (fi=0x555555aaf730, event=11, data=0x555555b189f0, file=0x7ffff7a3bc68 "sccp_scoc.c", line=1698) at fsm.c:872                 
#19 0x00007ffff7a19daa in sccp_scoc_rx_from_scrc (inst=0x555555a61b00, xua=0x555555b189f0) at sccp_scoc.c:1698                                                                  
#20 0x00007ffff7a150fd in scrc_rx_mtp_xfer_ind_xua (inst=0x555555a61b00, xua=0x555555b189f0) at sccp_scrc.c:479                                                                 
#21 0x00007ffff7a1ae48 in mtp_user_prim_cb (oph=0x555555af8cc8, ctx=0x555555a61b00) at sccp_user.c:202                                                                          
#22 0x00007ffff7a294a6 in deliver_to_mtp_user (osu=0x555555a61b48, xua=0x555555ad1120) at osmo_ss7_hmrt.c:95                                                                    
#23 0x00007ffff7a29673 in hmdt_message_for_distribution (inst=0x555555a2b3c0, xua=0x555555ad1120) at osmo_ss7_hmrt.c:134                                                        
#24 0x00007ffff7a2a0a7 in m3ua_hmdc_rx_from_l2 (inst=0x555555a2b3c0, xua=0x555555ad1120) at osmo_ss7_hmrt.c:278                                                                 
#25 0x00007ffff7a0b37f in m3ua_rx_xfer (asp=0x555555a602d0, xua=0x555555ad1120) at m3ua.c:577                                                                                   
#26 0x00007ffff7a0bb9b in m3ua_rx_msg (asp=0x555555a602d0, msg=0x555555adb750) at m3ua.c:732                                                                                    
#27 0x00007ffff7a27668 in xua_cli_read_cb (conn=0x555555a60bd0) at osmo_ss7.c:1950
#28 0x00007ffff7a96a3d in osmo_stream_cli_read (cli=0x555555a60bd0) at stream.c:327
#29 0x00007ffff7a9717f in osmo_stream_cli_fd_cb (ofd=0x555555a60bd0, what=1) at stream.c:446                                                                                    
#30 0x00007ffff7ac881c in poll_disp_fds (n_fd=12) at select.c:361
#31 0x00007ffff7ac8929 in _osmo_select_main (polling=0) at select.c:399
#32 0x00007ffff7ac8997 in osmo_select_main_ctx (polling=0) at select.c:455
#33 0x00005555555797f9 in main (argc=3, argv=0x7fffffffe4b8) at osmo_bsc_main.c:1043


(gdb) info locals
ep = 0x555555b2d350
fi = 0x555555b18960
cleared_ci = {ep = 0x555555b2d350, occupied = true, label = '\000' <repeats 63 times>, mgcp_client_fi = 0x0, pending = false, sent = false, verb = MGCP_VERB_DLCX, verb_info = {
    addr = '\000' <repeats 45 times>, port = 0, endpoint = '\000' <repeats 511 times>, call_id = 0, ptime = 0, codecs = {CODEC_PCMU_8000_1, CODEC_PCMU_8000_1,                  
      CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1}, codecs_len = 0,  
    ptmap = {{codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {              
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {                    
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}}, ptmap_len = 0, x_osmo_ign = 0, x_osmo_osmux_use = false, x_osmo_osmux_cid = 0,                
    conn_mode = MGCP_CONN_NONE, param_present = false, param = {amr_octet_aligned_present = false, amr_octet_aligned = false}}, notify = {entry = {next = 0x0, prev = 0x0},     
    fi = 0x0, success = 0, failure = 0, data = 0x0}, got_port_info = false, rtp_info = {addr = '\000' <repeats 45 times>, port = 0, endpoint = '\000' <repeats 511 times>,      
    call_id = 0, ptime = 0, codecs = {CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1,      
      CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1}, codecs_len = 0, ptmap = {{codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {            
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {                    
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}}, ptmap_len = 0,      
    x_osmo_ign = 0, x_osmo_osmux_use = false, x_osmo_osmux_cid = 0, conn_mode = MGCP_CONN_NONE, param_present = false, param = {amr_octet_aligned_present = false,              
      amr_octet_aligned = false}}, mgcp_ci_str = '\000' <repeats 32 times>}
(gdb) p ci
$1 = (struct osmo_mgcpc_ep_ci *) 0x555555b2d570
(gdb) p *ci
$2 = {ep = 0x555555b2d350, occupied = true, label = '\000' <repeats 63 times>, mgcp_client_fi = 0x0, pending = false, sent = false, verb = MGCP_VERB_DLCX, verb_info = {        
    addr = '\000' <repeats 45 times>, port = 0, endpoint = '\000' <repeats 511 times>, call_id = 0, ptime = 0, codecs = {CODEC_PCMU_8000_1, CODEC_PCMU_8000_1,                  
      CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1}, codecs_len = 0,  
    ptmap = {{codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {              
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {                    
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}}, ptmap_len = 0, x_osmo_ign = 0, x_osmo_osmux_use = false, x_osmo_osmux_cid = 0,                
    conn_mode = MGCP_CONN_NONE, param_present = false, param = {amr_octet_aligned_present = false, amr_octet_aligned = false}}, notify = {entry = {next = 0x0, prev = 0x0},     
    fi = 0x0, success = 0, failure = 0, data = 0x0}, got_port_info = false, rtp_info = {addr = '\000' <repeats 45 times>, port = 0, endpoint = '\000' <repeats 511 times>,      
    call_id = 0, ptime = 0, codecs = {CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1,      
      CODEC_PCMU_8000_1, CODEC_PCMU_8000_1, CODEC_PCMU_8000_1}, codecs_len = 0, ptmap = {{codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {            
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {                    
        codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}, {codec = CODEC_PCMU_8000_1, pt = 0}}, ptmap_len = 0,      
    x_osmo_ign = 0, x_osmo_osmux_use = false, x_osmo_osmux_cid = 0, conn_mode = MGCP_CONN_NONE, param_present = false, param = {amr_octet_aligned_present = false,              
      amr_octet_aligned = false}}, mgcp_ci_str = '\000' <repeats 32 times>}
(gdb) p ci->notify
$3 = {entry = {next = 0x0, prev = 0x0}, fi = 0x0, success = 0, failure = 0, data = 0x0}

Files

trace_540_fail_29072022.pcapng trace_540_fail_29072022.pcapng 5.08 KB dexter, 06/29/2022 04:10 PM
idb_dug20_25072022.dbb idb_dug20_25072022.dbb 54.8 KB dexter, 07/25/2022 11:03 AM
testnet_e1_at_sysmo.tar testnet_e1_at_sysmo.tar 50 KB dexter, 07/25/2022 03:12 PM
Hide closed

Checklist

  • TTCN3 reproducer for "540 FAIL" case
  • fix crash in "540 FAIL" situations
  • implement clearing all configured E1 trunks via wildcard DLCX on osmo-bsc start
Actions
Copy link

Also available in: Atom PDF

Add picture from clipboard (Maximum size: 48.8 MB)