Cellular Network Infrastructure » Core Network (CN) » OsmoSGSN

Full trace:

#0  rb_insert_color (node=node@entry=0xb7b862a0 <rate_ctr_timer>, root=root@entry=0xb7b84084 <timer_root>) at rbtree.c:80
        parent = 0x80bea8c
        gparent = 0x0
#1  0xb7b71576 in __add_timer (timer=0xb7b862a0 <rate_ctr_timer>) at timer.c:65
        new = <optimized out>
        parent = <optimized out>
#2  osmo_timer_add (timer=timer@entry=0xb7b862a0 <rate_ctr_timer>) at timer.c:76
No locals.
#3  0xb7b715db in osmo_timer_schedule (timer=timer@entry=0xb7b862a0 <rate_ctr_timer>, seconds=seconds@entry=1, 
    microseconds=microseconds@entry=0) at timer.c:98
        current_time = {tv_sec = 1467107894, tv_usec = 323237}
#4  0xb7b77e12 in rate_ctr_timer_cb (data=0x0) at rate_ctr.c:143
        ctrg = 0xb7b8404c <rate_ctr_groups>
#5  0xb7b71851 in osmo_timers_update () at timer.c:244
        current_time = {tv_sec = 1467107894, tv_usec = 323230}
        node = <optimized out>
        timer_eviction_list = {next = 0xbfffed18, prev = 0xbfffed18}
        this = <optimized out>
        work = 0
#6  0xb7b71e48 in osmo_select_main (polling=polling@entry=0) at select.c:188
        readset = {__fds_bits = {0 <repeats 32 times>}}
        writeset = {__fds_bits = {0 <repeats 32 times>}}
        exceptset = {__fds_bits = {0 <repeats 32 times>}}
        rc = <optimized out>
        no_time = {tv_sec = 0, tv_usec = 0}
#7  0x0804b2ba in main (argc=5, argv=0xbffff1b4) at sgsn_main.c:426
        ctrl = <optimized out>
        dummy_network = {country_code = 0, network_code = 0, name_long = 0xc <error: Cannot access memory at address 0xc>, 
          name_short = 0xb776977c "8\371\377\267`\310\375\267(\313", <incomplete sequence \375\267>, auth_policy = 3086906612, 
          authorized_regexp = {__buffer = 0xb776ded4 "", __allocated = 3078043188, __used = 0, __syntax = 0, 
            __fastmap = 0xb776977c "8\371\377\267`\310\375\267(\313", <incomplete sequence \375\267>, 
            __translate = 0x10 <error: Cannot access memory at address 0x10>, re_nsub = 3221221264, __can_be_null = 1, 
            __regs_allocated = 3, __fastmap_accurate = 1, __no_sub = 0, __not_bol = 1, __not_eol = 1, __newline_anchor = 1}, 
          authorized_reg_str = 0x7c9d3dea <error: Cannot access memory at address 0x7c9d3dea>, reject_cause = 3078020820, 
          a5_encryption = -1216926860, neci = 1, send_mm_info = 1398, handover = {active = -1216962880, win_rxlev_avg = 3081734952, 
            win_rxqual_avg = 3086904733, win_rxlev_avg_neigh = 3078082657, pwr_interval = 134519574, pwr_hysteresis = 0, 
            max_distance = 0}, stats = {chreq = {total = 0x1, no_channel = 0x4ca}, handover = {attempted = 0xb7769a00, 
              no_channel = 0x2, timeout = 0x804990b, completed = 0xb7af8adc, failed = 0xb7b8852c}, loc_upd_type = {
              attach = 0xb7fff000, normal = 0xb776dcc4, periodic = 0xc, detach = 0xb776977c}, loc_upd_resp = {
              reject = 0xb7fe74f4 <do_lookup_x+1764>, accept = 0xbffff0d4}, paging = {attempted = 0xb7fdcce0, detached = 0xbffff090, 
              completed = 0xb7fe777a <_dl_lookup_symbol_x+266>, expired = 0xb776977c}, sms = {submitted = 0x10, 
              no_receiver = 0xbffff010, delivered = 0x7b1ea71, rp_err_mem = 0xf63d4e2e, rp_err_other = 0xb776ded4}, call = {
              mo_setup = 0xb7776c34, mo_connect_ack = 0x0, mt_setup = 0x1, mt_connect = 0xb7fdcb28}, chan = {rf_fail = 0xbffff024, 
              rll_err = 0xbffff01c}, bts = {oml_fail = 0xb793733b, rsl_fail = 0xb7af85a8}}, mncc_state = 0x0, mncc_recv = 0x0, 
          upqueue = {next = 0x1, prev = 0x8d6}, trans_list = {next = 0xb77697e0, prev = 0xb7af8b28}, bsc_api = 0x8049a8a, 
          num_bts = 3078059124, bts_list = {next = 0x80485d8, prev = 0x1}, T3101 = 0, T3103 = 0, T3105 = -1, T3107 = -1207963648, 
          T3109 = -1073745580, T3111 = -1207960848, T3113 = -1073745648, T3115 = -1208060038, T3117 = -1073745728, T3119 = 134514136, 
          T3122 = -1073745720, T3141 = -1207960940, subscr_expire_timer = {node = {rb_parent_color = 0, rb_right = 0xb77697e0, 
              rb_left = 0x1}, list = {next = 0x0, prev = 0x1}, timeout = {tv_sec = -1207961288, tv_usec = 1}, active = 0, 
            cb = 0xb7fdcb28, data = 0x1}, rrlp = {mode = 3221221616}, ctype_by_chreq = {3086926177, 3086863584, GSM_LCHAN_NONE, 
            3221221716, 3221221576, 3221221568, 134519434, 3087006008, GSM_LCHAN_NONE, 3079753952, 3079753728, 4294967295, 
            GSM_LCHAN_SDCCH, 3078057012, 3078214387, 3079754700}, pag_any_tch = -1073745684, bsc_data = 0xb7fff938, 
          subscr_creation_mode = 134521673, ext_min = 13238735728016117760, ext_max = 578160124968230912, subscr_group = 0x5, 
          sms_queue = 0xbffff1b4, avoid_tmsi = -1073745460, ctrl = 0xb779d4ad <__cxa_atexit+29>}
        rc = <optimized out>

msuraev wrote:

Full trace:
dummy_network = {country_code = 0, network_code = 0, name_long = 0xc <error: Cannot access memory at address 0xc>,

a lot of the struct members are completely random / garbled, so I would presume there's a heap overrun somewhere. Did you try running it with valgrind?

Running sgsn under valgrind makes segfault disappear.

Closing as I'm unable to reproduce it.

Not sure if related but I've hit following segfault recently:

0002> gprs_gmm.c:2043 MM > DEACTIVATE PDP CONTEXT REQ (cause: Regular deactivation)
<000f> sgsn_libgtp.c:262 PDP Delete PDP Context
<0002> gprs_gmm.c:1726 MM < DEACTIVATE PDP CONTEXT ACK
<0012> gprs_llc_parse.c:74 LLC SAPI=3 C FCS=0x897927CMD=UI DATA
<0013> gprs_sndcp.c:538 Message for non-existing SNDCP Entity (lle=0x80c06a8, TLLI=c5bee9f6, SAPI=3, NSAPI=5)
<0012> gprs_llc_parse.c:74 LLC SAPI=3 C FCS=0xe6e7a2CMD=UI DATA
<0013> gprs_sndcp.c:538 Message for non-existing SNDCP Entity (lle=0x80c06a8, TLLI=c5bee9f6, SAPI=3, NSAPI=5)
<0012> gprs_llc_parse.c:74 LLC SAPI=3 C FCS=0xe1a13fCMD=UI DATA
<0013> gprs_sndcp.c:538 Message for non-existing SNDCP Entity (lle=0x80c06a8, TLLI=c5bee9f6, SAPI=3, NSAPI=5)
<0012> gprs_llc_parse.c:74 LLC SAPI=3 C FCS=0xdb5179CMD=UI DATA
<0013> gprs_sndcp.c:538 Message for non-existing SNDCP Entity (lle=0x80c06a8, TLLI=c5bee9f6, SAPI=3, NSAPI=5)
<0002> gprs_gmm.c:1699 MM <- DEACTIVATE PDP CONTEXT REQ
<0012> gprs_llc_parse.c:74 LLC SAPI=1 C FCS=0x0f457bCMD=UI DATA
<0002> gprs_gmm.c:1034 MM > GMM DETACH REQUEST TLLI=0xc5bee9f6 type=GPRS detach Power-off
<0002> gprs_gmm.c:170 MM Cleaning MM context due to GPRS DETACH REQUEST
<0002> gprs_gmm.c:1699 MM < DEACTIVATE PDP CONTEXT REQ
<0012> gprs_llc.c:165 LLC: unknown TLLI 0xc5bee9f6, creating LLME on the fly

Program received signal SIGSEGV, Segmentation fault.
0x0805985b in _bssgp_tx_dl_ud (mmctx=0x80c0ed0, msg=0x80c04a0) at gprs_llc.c:62
62 OSMO_ASSERT(msgb_tlli(msg) == mmctx->gb.llme->tlli
(gdb) bt
#0 0x0805985b in _bssgp_tx_dl_ud (mmctx=0x80c0ed0, msg=0x80c04a0) at gprs_llc.c:62
#1 gprs_llc_tx_ui (msg=msg@entry=0x80c04a0, sapi=sapi@entry=1 '\001', command=command@entry=0, mmctx=mmctx@entry=0x80c0ed0,
encryptable=encryptable@entry=true) at gprs_llc.c:466
#2 0x0804b94f in gsm48_gmm_sendmsg (msg=0x80c04a0, command=command@entry=0, mm=0x80c0ed0, encryptable=true) at gprs_gmm.c:141
#3 0x0804fca7 in _gsm48_tx_gsm_deact_pdp_req (sm_cause=<optimized out>, tid=<optimized out>, mm=<optimized out>) at gprs_gmm.c:1710
#4 gsm48_tx_gsm_deact_pdp_req (pdp=0x1, pdp@entry=0x80c1ce0, sm_cause=sm_cause@entry=38 '&') at gprs_gmm.c:1716
#5 0x0804fe10 in pdpctx_timer_cb (_pdp=0x80c1ce0) at gprs_gmm.c:2102
#6 0xb7b70821 in osmo_timers_update () at timer.c:244
#7 0xb7b70e18 in osmo_select_main (polling=polling@entry=0) at select.c:188
#8 0x0804b45d in main (argc=5, argv=0xbffff1b4) at sgsn_main.c:429

Full trace:
#0 0x0805985b in bssgp_tx_dl_ud (mmctx=0x80c0ed0, msg=0x80c04a0) at gprs_llc.c:62
dup = {tlli = 0x0, imsi = 0x80c0edc "", fc = 0x0, drx_parms = 0, ms_ra_cap = {len = 13, v = 0x80c0fca "\f\b\001"},
qos_profile = "\000\000"}
#1 gprs_llc_tx_ui (msg=msg@entry=0x80c04a0, sapi=sapi@entry=1 '\001', command=command@entry=0, mmctx=mmctx@entry=0x80c0ed0,
encryptable=encryptable@entry=true) at gprs_llc.c:466
lle = <optimized out>
fcs = 0x80c05e7 "\374mh"
addr = <optimized out>
ctrl = <optimized out>
fcs_calc = <optimized out>
nu = <optimized out>
oc = <optimized out>
#2 0x0804b94f in gsm48_gmm_sendmsg (msg=0x80c04a0, command=command@entry=0, mm=0x80c0ed0, encryptable=true) at gprs_gmm.c:141
No locals.
#3 0x0804fca7 in _gsm48_tx_gsm_deact_pdp_req (sm_cause=<optimized out>, tid=<optimized out>, mm=<optimized out>) at gprs_gmm.c:1710
gh = 0xbfffec44
transaction_id = <optimized out>
#4 gsm48_tx_gsm_deact_pdp_req (pdp=0x1, pdp@entry=0x80c1ce0, sm_cause=sm_cause@entry=38 '&') at gprs_gmm.c:1716
No locals.
#5 0x0804fe10 in pdpctx_timer_cb (_pdp=0x80c1ce0) at gprs_gmm.c:2102
pdp = 0x80c1ce0
#6 0xb7b70821 in osmo_timers_update () at timer.c:244
current_time = {tv_sec = 1470230876, tv_usec = 716156}
node = <optimized out>
timer_eviction_list = {next = 0xbfffed18, prev = 0xbfffed18}
this = <optimized out>
work = 0
#7 0xb7b70e18 in osmo_select_main (polling=polling@entry=0) at select.c:188
readset = {_fds_bits = {0 <repeats 32 times>}}
writeset = {__fds_bits = {0 <repeats 32 times>}}
exceptset = {__fds_bits = {0 <repeats 32 times>}}
rc = <optimized out>
no_time = {tv_sec = 0, tv_usec = 0}
#8 0x0804b45d in main (argc=5, argv=0xbffff1b4) at sgsn_main.c:429
ctrl = <optimized out>
dummy_network = {country_code = 0, network_code = 0,
name_long = 0xb776bc94 "\352=\235|\325\020\334", <incomplete sequence \304>,
name_short = 0xc <error: Cannot access memory at address 0xc>, auth_policy = 3077998460, authorized_regexp = {
_buffer = 0xb7fe74f4 <do_lookup_x+1764> "\205\300ub\213\006뼋\204$\220", __allocated = 3078016724, __used = 3078039092,
__syntax = 0, __fastmap = 0x0,
__translate = 0xb776877c "8\371\377\267`\310\375\267(\313", <incomplete sequence \375\267>, re_nsub = 16,
__can_be_null = 0, __regs_allocated = 0, __fastmap_accurate = 0, __no_sub = 1, __not_bol = 0, __not_eol = 0,
__newline_anchor = 1}, authorized_reg_str = 0x3e4e9ef <error: Cannot access memory at address 0x3e4e9ef>,
reject_cause = 2090679786, a5_encryption = -1216950572, neci = -1216930956, send_mm_info = 1, handover = {active = 1398,
win_rxlev_avg = 3078000320, win_rxqual_avg = 3081730856, win_rxlev_avg_neigh = 3086904733, pwr_interval = 3078078561,
pwr_hysteresis = 134519819, max_distance = 0}, stats = {chreq = {total = 0x0, no_channel = 0x1}, handover = {
attempted = 0x4ca, no_channel = 0xb7768a00, timeout = 0x2, completed = 0x80499ff, failed = 0xb7af7adc}, loc_upd_type = {
attach = 0xb7b8752c, normal = 0xb7fff000, periodic = 0xb776ccc4, detach = 0xc}, loc_upd_resp = {reject = 0xb776877c,
accept = 0xb7fe74f4 <do_lookup_x+1764>}, paging = {attempted = 0xbffff0d4, detached = 0xb7fdcce0,
completed = 0xbffff090, expired = 0xb7fe777a <_dl_lookup_symbol_x+266>}, sms = {submitted = 0xb776877c,
no_receiver = 0x10, delivered = 0xbffff010, rp_err_mem = 0x7b1ea71, rp_err_other = 0xf63d4e2e}, call = {
mo_setup = 0xb776ced4, mo_connect_ack = 0xb7775c34, mt_setup = 0x0, mt_connect = 0x1}, chan = {rf_fail = 0xb7fdcb28,
rll_err = 0xbffff024}, bts = {oml_fail = 0xbffff01c, rsl_fail = 0xb793633b}}, mncc_state = 0xb7af75a8, mncc_recv = 0x0,
upqueue = {next = 0x0, prev = 0x1}, trans_list = {next = 0x8d6, prev = 0xb77687e0}, bsc_api = 0xb7af7b28,
num_bts = 134519679, bts_list = {next = 0xb7776474, prev = 0x8048610}, T3101 = 1, T3103 = 0, T3105 = 0, T3107 = -1,
T3109 = -1207963648, T3111 = -1073745580, T3113 = -1207960848, T3115 = -1073745648, T3117 = -1208060038,
T3119 = -1073745728, T3122 = 134514192, T3141 = -1073745720, subscr_expire_timer = {node = {rb_parent_color = 3087006356,
rb_right = 0x0, rb_left = 0xb77687e0}, list = {next = 0x1, prev = 0x0}, timeout = {tv_sec = 1, tv_usec = -1207961288},
active = 1, cb = 0xb7fff000, data = 0xb7fdcb28}, rrlp = {mode = RRLP_MODE_MS_BASED}, ctype_by_chreq = {3221221616,
3086926177, 3086863584, GSM_LCHAN_NONE, 3221221716, 3221221576, 3221221568, 134519679, 3087006008, GSM_LCHAN_NONE,
3079749856, 3079749632, 4294967295, GSM_LCHAN_SDCCH, 3078052916, 3078210291}, pag_any_tch = -1215216692,
bsc_data = 0xbffff0ec, auto_create_subscr = 56, auto_assign_exten = 249, ext_min = 13235023773813744773,
ext_max = 578431080222777344, subscr_group = 0x8061922 <_libc_csu_init+82>, sms_queue = 0x5, avoid_tmsi = -1073745484,
ctrl = 0xbffff1cc, dyn_ts_allow_tch_f = 173}
rc = <optimized out>

I have not found a way to trigger it reliably yet.