Hi Daniel,

Here is the output.

(gdb) p *shm_block
$1 = {size = 268435456, used = 4889264, real_used = 5313920,
  max_real_used = 8245552, first_frag = 0x7f5394df3830,
  last_frag_end = 0x7f53a4de8ff0, free_hash = {{head = {size = 0, u = {
          nxt_free = 0x7f539524ed60, is_free = 139996961238368}}, tail = {
        size = 0, prev_free = 0x7f5395178340}, no = 29}, {head = {size = 0,
        u = {nxt_free = 0x7f53951cd4c0, is_free = 139996960707776}}, tail = {
        size = 0, prev_free = 0x7f53952e4170}, no = 65}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9080, is_free = 139996956627072}}, tail = {
        size = 0, prev_free = 0x7f5394de9080}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539537c110, is_free = 139996962472208}}, tail = {
        size = 0, prev_free = 0x7f53953b4470}, no = 307}, {head = {size = 0,
        u = {nxt_free = 0x7f5395220d30, is_free = 139996961049904}}, tail = {
        size = 0, prev_free = 0x7f53952a7410}, no = 125}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de90f8, is_free = 139996956627192}}, tail = {
        size = 0, prev_free = 0x7f5394de90f8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53951a1920, is_free = 139996960528672}}, tail = {
        size = 0, prev_free = 0x7f5395387aa0}, no = 29}, {head = {size = 0,
        u = {nxt_free = 0x7f53951cdc40, is_free = 139996960709696}}, tail = {
        size = 0, prev_free = 0x7f53954325f0}, no = 44}, {head = {size = 0,
        u = {nxt_free = 0x7f5395311110, is_free = 139996962033936}}, tail = {
        size = 0, prev_free = 0x7f5395175d20}, no = 48}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9198, is_free = 139996956627352}}, tail = {
        size = 0, prev_free = 0x7f5394de9198}, no = 0}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f5394de91c0, is_free = 139996956627392}}, tail = {
        size = 0, prev_free = 0x7f5394de91c0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de91e8, is_free = 139996956627432}}, tail = {
        size = 0, prev_free = 0x7f5394de91e8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9210, is_free = 139996956627472}}, tail = {
        size = 0, prev_free = 0x7f5394de9210}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53952b1920, is_free = 139996961642784}}, tail = {
        size = 0, prev_free = 0x7f539531ac30}, no = 27}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9260, is_free = 139996956627552}}, tail = {
        size = 0, prev_free = 0x7f5394de9260}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9288, is_free = 139996956627592}}, tail = {
        size = 0, prev_free = 0x7f5394de9288}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395231960, is_free = 139996961118560}}, tail = {
        size = 0, prev_free = 0x7f53952f4c10}, no = 20}, {head = {size = 0,
        u = {nxt_free = 0x7f5395401440, is_free = 139996963017792}}, tail = {
        size = 0, prev_free = 0x7f5395324f90}, no = 31}, {head = {size = 0,
        u = {nxt_free = 0x7f5395471770, is_free = 139996963477360}}, tail = {
        size = 0, prev_free = 0x7f539522a180}, no = 11}, {head = {size = 0,
        u = {nxt_free = 0x7f5395253e60, is_free = 139996961259104}}, tail = {
        size = 0, prev_free = 0x7f5395253e60}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9350, is_free = 139996956627792}}, tail = {
        size = 0, prev_free = 0x7f5394de9350}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539523afc0, is_free = 139996961157056}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f539540bd70}, no = 4}, {head = {size = 0,
        u = {nxt_free = 0x7f539528b7e0, is_free = 139996961486816}}, tail = {
        size = 0, prev_free = 0x7f53952602c0}, no = 6}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de93c8, is_free = 139996956627912}}, tail = {
        size = 0, prev_free = 0x7f5394de93c8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de93f0, is_free = 139996956627952}}, tail = {
        size = 0, prev_free = 0x7f5394de93f0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9418, is_free = 139996956627992}}, tail = {
        size = 0, prev_free = 0x7f5394de9418}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9440, is_free = 139996956628032}}, tail = {
        size = 0, prev_free = 0x7f5394de9440}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53951de260, is_free = 139996960776800}}, tail = {
        size = 0, prev_free = 0x7f53951de260}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9490, is_free = 139996956628112}}, tail = {
        size = 0, prev_free = 0x7f5394de9490}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de94b8, is_free = 139996956628152}}, tail = {
        size = 0, prev_free = 0x7f5394de94b8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395339790, is_free = 139996962199440}}, tail = {
        size = 0, prev_free = 0x7f5395339790}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5395485bc0, is_free = 139996963560384}}, tail = {
        size = 0, prev_free = 0x7f53953ee1d0}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f53953ba130, is_free = 139996962726192}}, tail = {
        size = 0, prev_free = 0x7f53952aa6e0}, no = 6}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f539525b350, is_free = 139996961289040}}, tail = {
        size = 0, prev_free = 0x7f53954711e0}, no = 40}, {head = {size = 0,
        u = {nxt_free = 0x7f539542b650, is_free = 139996963190352}}, tail = {
        size = 0, prev_free = 0x7f53951c06e0}, no = 6}, {head = {size = 0,
        u = {nxt_free = 0x7f53953b3340, is_free = 139996962698048}}, tail = {
        size = 0, prev_free = 0x7f53952b8330}, no = 11}, {head = {size = 0,
        u = {nxt_free = 0x7f539546daa0, is_free = 139996963461792}}, tail = {
        size = 0, prev_free = 0x7f53953b13f0}, no = 54}, {head = {size = 0,
        u = {nxt_free = 0x7f53952b98b0, is_free = 139996961675440}}, tail = {
        size = 0, prev_free = 0x7f53952bb0d0}, no = 98}, {head = {size = 0,
        u = {nxt_free = 0x7f53950de6b0, is_free = 139996959729328}}, tail = {
        size = 0, prev_free = 0x7f539526d3a0}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5395485dd0, is_free = 139996963560912}}, tail = {
        size = 0, prev_free = 0x7f53953030c0}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f53953fe1c0, is_free = 139996963004864}}, tail = {
        size = 0, prev_free = 0x7f5395345870}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f53951ff540, is_free = 139996960912704}}, tail = {
        size = 0, prev_free = 0x7f5395368ad0}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5395235d10, is_free = 139996961135888}}, tail = {
        size = 0, prev_free = 0x7f539546e180}, no = 7}, {head = {size = 0,
        u = {nxt_free = 0x7f53952e7dd0, is_free = 139996961865168}}, tail = {
        size = 0, prev_free = 0x7f53954602a0}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9710, is_free = 139996956628752}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394de9710}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9738, is_free = 139996956628792}}, tail = {
        size = 0, prev_free = 0x7f5394de9738}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53954b00d0, is_free = 139996963733712}}, tail = {
        size = 0, prev_free = 0x7f53954d4790}, no = 5}, {head = {size = 0,
        u = {nxt_free = 0x7f539536b9b0, is_free = 139996962404784}}, tail = {
        size = 0, prev_free = 0x7f5395334930}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f539537c720, is_free = 139996962473760}}, tail = {
        size = 0, prev_free = 0x7f539528bc30}, no = 14}, {head = {size = 0,
        u = {nxt_free = 0x7f5395339460, is_free = 139996962198624}}, tail = {
        size = 0, prev_free = 0x7f5395353dd0}, no = 8}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9800, is_free = 139996956628992}}, tail = {
        size = 0, prev_free = 0x7f5394de9800}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395432180, is_free = 139996963217792}}, tail = {
        size = 0, prev_free = 0x7f5395432180}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f539528cc00, is_free = 139996961491968}}, tail = {
        size = 0, prev_free = 0x7f5395210520}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9878, is_free = 139996956629112}}, tail = {
        size = 0, prev_free = 0x7f5394de9878}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395206a20, is_free = 139996960942624}}, tail = {
        size = 0, prev_free = 0x7f5395206a20}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de98c8, is_free = 139996956629192}}, tail = {
        size = 0, prev_free = 0x7f5394de98c8}, no = 0}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f5394de98f0, is_free = 139996956629232}}, tail = {
        size = 0, prev_free = 0x7f5394de98f0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9918, is_free = 139996956629272}}, tail = {
        size = 0, prev_free = 0x7f5394de9918}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9940, is_free = 139996956629312}}, tail = {
        size = 0, prev_free = 0x7f5394de9940}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9968, is_free = 139996956629352}}, tail = {
        size = 0, prev_free = 0x7f5394de9968}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9990, is_free = 139996956629392}}, tail = {
        size = 0, prev_free = 0x7f5394de9990}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de99b8, is_free = 139996956629432}}, tail = {
        size = 0, prev_free = 0x7f5394de99b8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de99e0, is_free = 139996956629472}}, tail = {
        size = 0, prev_free = 0x7f5394de99e0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9a08, is_free = 139996956629512}}, tail = {
        size = 0, prev_free = 0x7f5394de9a08}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9a30, is_free = 139996956629552}}, tail = {
        size = 0, prev_free = 0x7f5394de9a30}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9a58, is_free = 139996956629592}}, tail = {
        size = 0, prev_free = 0x7f5394de9a58}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9a80, is_free = 139996956629632}}, tail = {
        size = 0, prev_free = 0x7f5394de9a80}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9aa8, is_free = 139996956629672}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394de9aa8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9ad0, is_free = 139996956629712}}, tail = {
        size = 0, prev_free = 0x7f5394de9ad0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9af8, is_free = 139996956629752}}, tail = {
        size = 0, prev_free = 0x7f5394de9af8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539530a5d0, is_free = 139996962006480}}, tail = {
        size = 0, prev_free = 0x7f539530a5d0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9b48, is_free = 139996956629832}}, tail = {
        size = 0, prev_free = 0x7f5394de9b48}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395410d00, is_free = 139996963081472}}, tail = {
        size = 0, prev_free = 0x7f53954ce130}, no = 10}, {head = {size = 0,
        u = {nxt_free = 0x7f53951b3380, is_free = 139996960600960}}, tail = {
        size = 0, prev_free = 0x7f5395419370}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f53953d9c30, is_free = 139996962855984}}, tail = {
        size = 0, prev_free = 0x7f5395430a00}, no = 20}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9be8, is_free = 139996956629992}}, tail = {
        size = 0, prev_free = 0x7f5394de9be8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539543b400, is_free = 139996963255296}}, tail = {
        size = 0, prev_free = 0x7f53954362d0}, no = 12}, {head = {size = 0,
        u = {nxt_free = 0x7f539521f2d0, is_free = 139996961043152}}, tail = {
        size = 0, prev_free = 0x7f539531a740}, no = 51}, {head = {size = 0,
        u = {nxt_free = 0x7f53954d1db0, is_free = 139996963872176}}, tail = {
        size = 0, prev_free = 0x7f53954ef540}, no = 326}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f53954bd450, is_free = 139996963787856}}, tail = {
        size = 0, prev_free = 0x7f53954227e0}, no = 10}, {head = {size = 0,
        u = {nxt_free = 0x7f53954154c0, is_free = 139996963099840}}, tail = {
        size = 0, prev_free = 0x7f53954cc7f0}, no = 12}, {head = {size = 0,
        u = {nxt_free = 0x7f539531d6e0, is_free = 139996962084576}}, tail = {
        size = 0, prev_free = 0x7f5395476b50}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f5395427580, is_free = 139996963173760}}, tail = {
        size = 0, prev_free = 0x7f5395205410}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f53953dc290, is_free = 139996962865808}}, tail = {
        size = 0, prev_free = 0x7f53954869a0}, no = 10}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9d50, is_free = 139996956630352}}, tail = {
        size = 0, prev_free = 0x7f5394de9d50}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9d78, is_free = 139996956630392}}, tail = {
        size = 0, prev_free = 0x7f5394de9d78}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395550770, is_free = 139996964390768}}, tail = {
        size = 0, prev_free = 0x7f53953a7ce0}, no = 5}, {head = {size = 0,
        u = {nxt_free = 0x7f53952257e0, is_free = 139996961069024}}, tail = {
        size = 0, prev_free = 0x7f53952257e0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f539534fcc0, is_free = 139996962290880}}, tail = {
        size = 0, prev_free = 0x7f539528bf50}, no = 6}, {head = {size = 0,
        u = {nxt_free = 0x7f539529f880, is_free = 139996961568896}}, tail = {
        size = 0, prev_free = 0x7f539529f880}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f539544bd70, is_free = 139996963323248}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f539536f510}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f539544d230, is_free = 139996963328560}}, tail = {
        size = 0, prev_free = 0x7f53951de920}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f53953537f0, is_free = 139996962306032}}, tail = {
        size = 0, prev_free = 0x7f53955caa40}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f53952d99f0, is_free = 139996961806832}}, tail = {
        size = 0, prev_free = 0x7f53952d99f0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9ee0, is_free = 139996956630752}}, tail = {
        size = 0, prev_free = 0x7f5394de9ee0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53954d0380, is_free = 139996963865472}}, tail = {
        size = 0, prev_free = 0x7f53954d0380}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5395374320, is_free = 139996962439968}}, tail = {
        size = 0, prev_free = 0x7f5395374320}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f53951ffb30, is_free = 139996960914224}}, tail = {
        size = 0, prev_free = 0x7f53951ffb30}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f53954d53d0, is_free = 139996963886032}}, tail = {
        size = 0, prev_free = 0x7f5395221e10}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9fa8, is_free = 139996956630952}}, tail = {
        size = 0, prev_free = 0x7f5394de9fa8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394de9fd0, is_free = 139996956630992}}, tail = {
        size = 0, prev_free = 0x7f5394de9fd0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395329030, is_free = 139996962132016}}, tail = {
        size = 0, prev_free = 0x7f53952191f0}, no = 2}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f53954b42e0, is_free = 139996963750624}}, tail = {
        size = 0, prev_free = 0x7f53954b42e0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea048, is_free = 139996956631112}}, tail = {
        size = 0, prev_free = 0x7f5394dea048}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5395471b20, is_free = 139996963478304}}, tail = {
        size = 0, prev_free = 0x7f5395471b20}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea098, is_free = 139996956631192}}, tail = {
        size = 0, prev_free = 0x7f5394dea098}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea0c0, is_free = 139996956631232}}, tail = {
        size = 0, prev_free = 0x7f5394dea0c0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea0e8, is_free = 139996956631272}}, tail = {
        size = 0, prev_free = 0x7f5394dea0e8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea110, is_free = 139996956631312}}, tail = {
        size = 0, prev_free = 0x7f5394dea110}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea138, is_free = 139996956631352}}, tail = {
        size = 0, prev_free = 0x7f5394dea138}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea160, is_free = 139996956631392}}, tail = {
        size = 0, prev_free = 0x7f5394dea160}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea188, is_free = 139996956631432}}, tail = {
        size = 0, prev_free = 0x7f5394dea188}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea1b0, is_free = 139996956631472}}, tail = {
        size = 0, prev_free = 0x7f5394dea1b0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea1d8, is_free = 139996956631512}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394dea1d8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53952ad000, is_free = 139996961624064}}, tail = {
        size = 0, prev_free = 0x7f53952ad000}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea228, is_free = 139996956631592}}, tail = {
        size = 0, prev_free = 0x7f5394dea228}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539554c700, is_free = 139996964374272}}, tail = {
        size = 0, prev_free = 0x7f53954465f0}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea278, is_free = 139996956631672}}, tail = {
        size = 0, prev_free = 0x7f5394dea278}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53955cb020, is_free = 139996964892704}}, tail = {
        size = 0, prev_free = 0x7f53955cb020}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea2c8, is_free = 139996956631752}}, tail = {
        size = 0, prev_free = 0x7f5394dea2c8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea2f0, is_free = 139996956631792}}, tail = {
        size = 0, prev_free = 0x7f5394dea2f0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea318, is_free = 139996956631832}}, tail = {
        size = 0, prev_free = 0x7f5394dea318}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea340, is_free = 139996956631872}}, tail = {
        size = 0, prev_free = 0x7f5394dea340}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea368, is_free = 139996956631912}}, tail = {
        size = 0, prev_free = 0x7f5394dea368}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea390, is_free = 139996956631952}}, tail = {
        size = 0, prev_free = 0x7f5394dea390}, no = 0}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f5394dea3b8, is_free = 139996956631992}}, tail = {
        size = 0, prev_free = 0x7f5394dea3b8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea3e0, is_free = 139996956632032}}, tail = {
        size = 0, prev_free = 0x7f5394dea3e0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea408, is_free = 139996956632072}}, tail = {
        size = 0, prev_free = 0x7f5394dea408}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea430, is_free = 139996956632112}}, tail = {
        size = 0, prev_free = 0x7f5394dea430}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea458, is_free = 139996956632152}}, tail = {
        size = 0, prev_free = 0x7f5394dea458}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea480, is_free = 139996956632192}}, tail = {
        size = 0, prev_free = 0x7f5394dea480}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea4a8, is_free = 139996956632232}}, tail = {
        size = 0, prev_free = 0x7f5394dea4a8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea4d0, is_free = 139996956632272}}, tail = {
        size = 0, prev_free = 0x7f5394dea4d0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea4f8, is_free = 139996956632312}}, tail = {
        size = 0, prev_free = 0x7f5394dea4f8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea520, is_free = 139996956632352}}, tail = {
        size = 0, prev_free = 0x7f5394dea520}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f53951f6220, is_free = 139996960875040}}, tail = {
        size = 0, prev_free = 0x7f53951f6220}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea570, is_free = 139996956632432}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394dea570}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539523d3c0, is_free = 139996961166272}}, tail = {
        size = 0, prev_free = 0x7f539523d3c0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f53952138e0, is_free = 139996960995552}}, tail = {
        size = 0, prev_free = 0x7f53951f2350}, no = 4}, {head = {size = 0,
        u = {nxt_free = 0x7f539544e700, is_free = 139996963333888}}, tail = {
        size = 0, prev_free = 0x7f539526fe70}, no = 4}, {head = {size = 0,
        u = {nxt_free = 0x7f5395298610, is_free = 139996961539600}}, tail = {
        size = 0, prev_free = 0x7f53955e7950}, no = 4}, {head = {size = 0,
        u = {nxt_free = 0x7f53954bd960, is_free = 139996963789152}}, tail = {
        size = 0, prev_free = 0x7f53954bd960}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5395480560, is_free = 139996963538272}}, tail = {
        size = 0, prev_free = 0x7f5395369260}, no = 4}, {head = {size = 0,
        u = {nxt_free = 0x7f53952f85b0, is_free = 139996961932720}}, tail = {
        size = 0, prev_free = 0x7f539524b1a0}, no = 3}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea6b0, is_free = 139996956632752}}, tail = {
        size = 0, prev_free = 0x7f5394dea6b0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539553a230, is_free = 139996964299312}}, tail = {
        size = 0, prev_free = 0x7f539543a5f0}, no = 47}, {head = {size = 0,
        u = {nxt_free = 0x7f53955fa230, is_free = 139996965085744}}, tail = {
        size = 0, prev_free = 0x7f53953a2b10}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f5395462be0, is_free = 139996963417056}}, tail = {
        size = 0, prev_free = 0x7f53951b98d0}, no = 7}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f53951fe260, is_free = 139996960907872}}, tail = {
        size = 0, prev_free = 0x7f53953e2c20}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f5395300330, is_free = 139996961964848}}, tail = {
        size = 0, prev_free = 0x7f53953658d0}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea7a0, is_free = 139996956632992}}, tail = {
        size = 0, prev_free = 0x7f5394dea7a0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f539522bd30, is_free = 139996961094960}}, tail = {
        size = 0, prev_free = 0x7f539522bd30}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f53954c5ea0, is_free = 139996963823264}}, tail = {
        size = 0, prev_free = 0x7f539522c6c0}, no = 2}, {head = {size = 0,
        u = {nxt_free = 0x7f53953c94e0, is_free = 139996962788576}}, tail = {
        size = 0, prev_free = 0x7f53953c94e0}, no = 1}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea840, is_free = 139996956633152}}, tail = {
        size = 0, prev_free = 0x7f5394dea840}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea868, is_free = 139996956633192}}, tail = {
        size = 0, prev_free = 0x7f5394dea868}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea890, is_free = 139996956633232}}, tail = {
        size = 0, prev_free = 0x7f5394dea890}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea8b8, is_free = 139996956633272}}, tail = {
        size = 0, prev_free = 0x7f5394dea8b8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea8e0, is_free = 139996956633312}}, tail = {
        size = 0, prev_free = 0x7f5394dea8e0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea908, is_free = 139996956633352}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394dea908}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea930, is_free = 139996956633392}}, tail = {
        size = 0, prev_free = 0x7f5394dea930}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea958, is_free = 139996956633432}}, tail = {
        size = 0, prev_free = 0x7f5394dea958}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea980, is_free = 139996956633472}}, tail = {
        size = 0, prev_free = 0x7f5394dea980}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea9a8, is_free = 139996956633512}}, tail = {
        size = 0, prev_free = 0x7f5394dea9a8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea9d0, is_free = 139996956633552}}, tail = {
        size = 0, prev_free = 0x7f5394dea9d0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dea9f8, is_free = 139996956633592}}, tail = {
        size = 0, prev_free = 0x7f5394dea9f8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaa20, is_free = 139996956633632}}, tail = {
        size = 0, prev_free = 0x7f5394deaa20}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaa48, is_free = 139996956633672}}, tail = {
        size = 0, prev_free = 0x7f5394deaa48}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaa70, is_free = 139996956633712}}, tail = {
        size = 0, prev_free = 0x7f5394deaa70}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaa98, is_free = 139996956633752}}, tail = {
        size = 0, prev_free = 0x7f5394deaa98}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaac0, is_free = 139996956633792}}, tail = {
        size = 0, prev_free = 0x7f5394deaac0}, no = 0}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f5394deaae8, is_free = 139996956633832}}, tail = {
        size = 0, prev_free = 0x7f5394deaae8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deab10, is_free = 139996956633872}}, tail = {
        size = 0, prev_free = 0x7f5394deab10}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deab38, is_free = 139996956633912}}, tail = {
        size = 0, prev_free = 0x7f5394deab38}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deab60, is_free = 139996956633952}}, tail = {
        size = 0, prev_free = 0x7f5394deab60}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deab88, is_free = 139996956633992}}, tail = {
        size = 0, prev_free = 0x7f5394deab88}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deabb0, is_free = 139996956634032}}, tail = {
        size = 0, prev_free = 0x7f5394deabb0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deabd8, is_free = 139996956634072}}, tail = {
        size = 0, prev_free = 0x7f5394deabd8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deac00, is_free = 139996956634112}}, tail = {
        size = 0, prev_free = 0x7f5394deac00}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deac28, is_free = 139996956634152}}, tail = {
        size = 0, prev_free = 0x7f5394deac28}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deac50, is_free = 139996956634192}}, tail = {
        size = 0, prev_free = 0x7f5394deac50}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deac78, is_free = 139996956634232}}, tail = {
        size = 0, prev_free = 0x7f5394deac78}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaca0, is_free = 139996956634272}}, tail = {
---Type <return> to continue, or q <return> to quit---
        size = 0, prev_free = 0x7f5394deaca0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deacc8, is_free = 139996956634312}}, tail = {
        size = 0, prev_free = 0x7f5394deacc8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deacf0, is_free = 139996956634352}}, tail = {
        size = 0, prev_free = 0x7f5394deacf0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dead18, is_free = 139996956634392}}, tail = {
        size = 0, prev_free = 0x7f5394dead18}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dead40, is_free = 139996956634432}}, tail = {
        size = 0, prev_free = 0x7f5394dead40}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dead68, is_free = 139996956634472}}, tail = {
        size = 0, prev_free = 0x7f5394dead68}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394dead90, is_free = 139996956634512}}, tail = {
        size = 0, prev_free = 0x7f5394dead90}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deadb8, is_free = 139996956634552}}, tail = {
        size = 0, prev_free = 0x7f5394deadb8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deade0, is_free = 139996956634592}}, tail = {
        size = 0, prev_free = 0x7f5394deade0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deae08, is_free = 139996956634632}}, tail = {
        size = 0, prev_free = 0x7f5394deae08}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deae30, is_free = 139996956634672}}, tail = {
        size = 0, prev_free = 0x7f5394deae30}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deae58, is_free = 139996956634712}}, tail = {
        size = 0, prev_free = 0x7f5394deae58}, no = 0}, {head = {size = 0,
---Type <return> to continue, or q <return> to quit---
        u = {nxt_free = 0x7f5394deae80, is_free = 139996956634752}}, tail = {
        size = 0, prev_free = 0x7f5394deae80}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaea8, is_free = 139996956634792}}, tail = {
        size = 0, prev_free = 0x7f5394deaea8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaed0, is_free = 139996956634832}}, tail = {
        size = 0, prev_free = 0x7f5394deaed0}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaef8, is_free = 139996956634872}}, tail = {
        size = 0, prev_free = 0x7f5394deaef8}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaf20, is_free = 139996956634912}}, tail = {
        size = 0, prev_free = 0x7f5394deaf20}, no = 0}, {head = {size = 0,
        u = {nxt_free = 0x7f5394deaf48, is_free = 139996956634952}}, tail = {
        size = 0, prev_free = 0x7f5394deaf48}, no = 0}...}}
(gdb)




On Tue, Oct 29, 2013 at 9:56 AM, Daniel-Constantin Mierla <miconda@gmail.com> wrote:
Hello,

going to look over it. Do you still have the core file? If yes, provide the output of:

p *shm_block

from inside gdb.

Cheers,
Daniel


On 10/28/13 1:00 PM, Morten Isaksen wrote:
Hi,

We upgraded to 4.0.4 and now we got a new core dump in q_malloc.c:269. It has been stable for about 5 days after the upgrade.

/Morten

The full backtrace is here:

Loaded symbols for /lib64/libnss_files.so.2
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio/kamailio.pid -m 256 -M 8 -u kamai'.
Program terminated with signal 11, Segmentation fault.
#0  qm_detach_free (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:269
269             FRAG_END(next)->prev_free=prev;
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.107.el6.x86_64 hiredis-0.10.1-3.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6.x86_64 libcom_err-1.41.12-14.el6.x86_64 libselinux-2.0.94-5.3.el6.x86_64 libxml2-2.7.6-12.el6_4.1.x86_64 mysql-libs-5.1.67-1.el6_3.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64 openssl-1.0.0-27.el6_4.2.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0  qm_detach_free (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:269
        prev = 0x7f539524ed60
        next = 0x0
#1  qm_malloc (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:386
        f = 0x7f539524ed60
        hash = 0
#2  0x00007f53a5662950 in shm_malloc (size=0) at ../../mem/shm_mem.h:262
        p = <value optimized out>
#3  0x00007f53a566b49b in dlg_set_leg_info (dlg=<value optimized out>, tag=0x7ffff680b9a0, rr=0x7ffff680b860, contact=0x7ffff680b870, cseq=0x7ffff680b880, leg=1) at dlg_hash.c:498
        p = <value optimized out>
        __FUNCTION__ = "dlg_set_leg_info"
#4  0x00007f53a5659e88 in populate_leg_info (dlg=0x7f53952312b0, msg=<value optimized out>, t=<value optimized out>, leg=1, tag=0x7ffff680b9a0) at dlg_handlers.c:276
        skip_recs = 4
        cseq = {s = 0x0, len = 0}
        contact = {
          s = 0x8da2c0 "sip:65993777@178.21.249.22:5060>\r\nContent-Type: application/sdp\r\nRequire: timer\r\nContent-Length: 252\r\n\r\nv=0\r\no=- 2042480911 2042480911 IN IP4 178.21.249.22\r\ns=Hosted PBX\r\nc=IN IP4 178.21.249.22\r\nt=0 0"..., len = 31}
        rr_set = {s = 0x7f53ab927c20  <incomplete sequence \361\237\215>, len = 0}
        __FUNCTION__ = "populate_leg_info"
#5  0x00007f53a565b949 in dlg_onreply (t=0x7f5395245030, type=<value optimized out>, param=<value optimized out>) at dlg_handlers.c:500
        dlg = 0x7f53952312b0
        iuid = <value optimized out>
        new_state = 3
        old_state = 1
        unref = 0
        event = 3
        tag = {
          s = 0x8da1bc "as77c791c9\r\nCall-ID: 0dd71e51-ba5d-1231-c79f-00095200eb1e\r\nCSeq: 1206140 INVITE\r\nServer: one-connect\r\nAllow: INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, SUBSCRIBE, NOTIFY, INFO, PUBLISH\r\nSupported: repl"..., len = 10}
        req = 0x7f5395640090
        rpl = 0x7f53ab8cd7a0
        __FUNCTION__ = "dlg_onreply"
#6  0x00007f53aa1f58ad in run_trans_callbacks_internal (cb_lst=<value optimized out>, type=1048576, trans=0x7f5395245030, params=0x7ffff680baa0) at t_hooks.c:290
        cbp = 0x7f5395311260
        backup_from = 0x8e9ed0
        backup_to = 0x8e9ed8
        backup_dom_from = 0x8e9ee0
        backup_dom_to = 0x8e9ee8
        backup_uri_from = 0x8e9ec0
        backup_uri_to = 0x8e9ec8
        backup_xavps = 0x8e9fd0
        __FUNCTION__ = "run_trans_callbacks_internal"
#7  0x00007f53aa1f5a94 in run_trans_callbacks_with_buf (type=<value optimized out>, rbuf=0x7f53952450f0, req=<value optimized out>, repl=<value optimized out>, flags=<value optimized out>) at t_hooks.c:336
        params = {req = 0x7f5395640090, rpl = 0x7f53ab8cd7a0, param = 0x7f5395311270, code = 200, flags = 200, branch = 0, t_rbuf = 0x7f53952450f0, dst = 0x7f5395245140, send_buf = {
            s = 0x7f53953ac460 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UDP 178.21.248.8;rport=5060;branch=z9hG4bKd24f.b20f4bb3.0\r\nVia: SIP/2.0/UDP 178.21.248.20;branch=z"..., len = 1341}}
        trans = <value optimized out>
#8  0x00007f53aa22728c in relay_reply (t=<value optimized out>, p_msg=<value optimized out>, branch=0, msg_status=200, cancel_data=0x7ffff680be40, do_put_on_wait=1) at t_reply.c:1867
        relay = 0
        save_clone = 0
        buf = 0x7f53ab9361a0 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UDP 178.21.248.8;rport=5060;branch=z9hG4bKd24f.b20f4bb3.0\r\nVia: SIP/2.0/UDP 178.21.248.20;branch=z"...
        res_len = 1341
        relayed_code = 200
        relayed_msg = 0x7f53ab8cd7a0
        reply_bak = <value optimized out>
        bm = {to_tag_val = {s = 0x7f5395600c50 "P\233\365\224S\177", len = -1440728525}}
        totag_retr = 0
        reply_status = RPS_COMPLETED
        uas_rb = <value optimized out>
        to_tag = <value optimized out>
        reason = {s = 0x3 <Address 0x3 out of bounds>, len = -159334788}
        onsend_params = {req = 0x7f53aa45e594, rpl = 0x7ffff680be78, param = 0x0, code = 0, flags = 0, branch = 0, t_rbuf = 0x10, dst = 0x7f53aa200a91, send_buf = {s = 0x7f5395601f38 "", len = -1440601726}}
---Type <return> to continue, or q <return> to quit---
        __FUNCTION__ = "relay_reply"
#9  0x00007f53aa227da6 in reply_received (p_msg=0x7f53ab8cd7a0) at t_reply.c:2374
        msg_status = 200
        last_uac_status = 100
        ack = 0x1 <Address 0x1 out of bounds>
        ack_len = 0
        branch = 0
        reply_status = <value optimized out>
        onreply_route = <value optimized out>
        cancel_data = {cancel_bitmap = 0, reason = {cause = 200, u = {text = {s = 0x0, len = -1416571248}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = -1416571248}}}}
        uac = 0x7f5395245198
        t = 0x7f5395245030
        lack_dst = {send_sock = 0x0, to = {s = {sa_family = 48720, sa_data = "\200\366\377\177\000\000\000\000\000\000\000\000\000"}, sin = {sin_family = 48720, sin_port = 63104, sin_addr = {
                s_addr = 32767}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 48720, sin6_port = 63104, sin6_flowinfo = 32767, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\000\000\000\000\000\000\000\000\260kq\253S\177\000", __u6_addr16 = {0, 0, 0, 0, 27568, 43889, 32595, 0}, __u6_addr32 = {0, 0, 2876337072, 32595}}},
              sin6_scope_id = 2878134176}}, id = 32595, proto = -76 '\264', send_flags = {f = 5 '\005', blst_imask = 0 '\000'}}
        backup_user_from = <value optimized out>
        backup_user_to = <value optimized out>
        backup_domain_from = <value optimized out>
        backup_domain_to = <value optimized out>
        backup_uri_from = <value optimized out>
        backup_uri_to = <value optimized out>
        backup_xavps = <value optimized out>
        replies_locked = <value optimized out>
        branch_ret = <value optimized out>
        prev_branch = <value optimized out>
        blst_503_timeout = <value optimized out>
        hf = <value optimized out>
        onsend_params = {req = 0x8da424, rpl = 0x543274, param = 0x10, code = -1416833120, flags = 32595, branch = 0, t_rbuf = 0x7ffff680be50, dst = 0x7f53a566690b, send_buf = {
            s = 0x4ff1e7b00000000 <Address 0x4ff1e7b00000000 out of bounds>, len = 0}}
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140737329020144, -5577311537021373234, 139997336902320, 139997337147296, 200, 139996961202968, 5577292203380126926,
                -5577312076254333746}, __mask_was_saved = 0, __saved_mask = {__val = {139997337420128, 9281488, 139997335445253, 1, 140737329020496, 6191275, 5978073, 8591616, 9281252, 68728758110,
                  5548121, 9281656, 9281173, 139997337409168, 9282596, 139997337519904}}}}}
        __FUNCTION__ = "reply_received"
#10 0x0000000000456d54 in do_forward_reply (msg=0x7f53ab8cd7a0, mode=<value optimized out>) at forward.c:799
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
            sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}},
              sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = <value optimized out>
        r = <value optimized out>
        s = <value optimized out>
        len = <value optimized out>
        __FUNCTION__ = "do_forward_reply"
#11 0x000000000049f4be in receive_msg (
    buf=0x8d9e80 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"..., len=<value optimized out>, rcv_info=0x7ffff680c100) at receive.c:272
        msg = 0x7f53ab8cd7a0
        ctx = {rec_lev = 12, run_flags = 0, last_retcode = -1517756399, jmp_env = {{__jmpbuf = {139996960369776, 12, 219309716216, 139997343567872, 140737329021424, 4294967295, 140737329021551, 1},
              __mask_was_saved = 8581896, __saved_mask = {__val = {0, 28, 16, 0, 219305533392, 1, 0, 139997335309296, 219309716216, 139996960369776, 139997337398864, 139997343565480, 139997337398872,
                  140737329021320, 219305559701, 140737329021192}}}}}
        ret = <value optimized out>
        inb = {
          s = 0x8d9e80 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"..., len = 1444}
        __FUNCTION__ = "receive_msg"
#12 0x0000000000532306 in udp_rcv_loop () at udp_server.c:557
        len = 1444
        buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"...
        from = 0x7f53ab90ae50
        fromlen = 16
        ri = {src_ip = {af = 2, len = 4, u = {addrl = {385422770, 0}, addr32 = {385422770, 0, 0, 0}, addr16 = {5554, 5881, 0, 0, 0, 0, 0, 0}, addr = "\262\025\371\026", '\000' <repeats 11 times>}},
---Type <return> to continue, or q <return> to quit---
          dst_ip = {af = 2, len = 4, u = {addrl = {351868338, 0}, addr32 = {351868338, 0, 0, 0}, addr16 = {5554, 5369, 0, 0, 0, 0, 0, 0}, addr = "\262\025\371\024", '\000' <repeats 11 times>}},
          src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\023IJ\025\371\026\000\000\000\000\000\000\000"}, sin = {sin_family = 2,
              sin_port = 50195, sin_addr = {s_addr = 385422770}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 385422770, sin6_addr = {__in6_u = {
                  __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x7f53ab8cec10, proto = 1 '\001'}
        __FUNCTION__ = "udp_rcv_loop"
#13 0x0000000000467a7a in main_loop () at main.c:1638
        i = <value optimized out>
        pid = <value optimized out>
        si = <value optimized out>
        si_desc = "udp receiver child=3 sock=178.21.249.20:5060\000\000\000\000@\\\220\253S\177\000\000\000\000\000\000\000\000\000\000\003\000\000\000\000\000\000\000\001\000\000\000\001\000\000\000\300\375\216\000\000\000\000\000\001\000\000\000\000\000\000\000\000\376\216\000\000\000\000\000\000\000\200\020", '\000' <repeats 12 times>, "\005\000\000\000\000\000\000"
        nrprocs = <value optimized out>
        __FUNCTION__ = "main_loop"
#14 0x000000000046a912 in main (argc=<value optimized out>, argv=<value optimized out>) at main.c:2566
        cfg_stream = <value optimized out>
        c = <value optimized out>
        r = <value optimized out>
        tmp = 0x7ffff680d89a ""
        tmp_len = 0
        options = 0x5c1de8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1043769636
        rfd = <value optimized out>
        debug_save = <value optimized out>
        debug_flag = <value optimized out>
        dont_fork_cnt = <value optimized out>
        n_lst = <value optimized out>
        p = <value optimized out>
        __FUNCTION__ = "main"



On Mon, Oct 21, 2013 at 8:15 PM, Daniel-Constantin Mierla <miconda@gmail.com> wrote:
Hello,

can you upgrade to 4.0.4? there were some fixes related to new fields in sip message structure that resulted in some out bound memory inside transactions.

The config file and database is the same, so just re-install over the previous installation.

Cheers,
Daniel


On 10/21/13 11:10 AM, Morten Isaksen wrote:
Hi,

Our Kamailio has stopped with a segmentation fault 4 times the last week.

There has been no changes to the configuration file the last 15 days, so I suspect a SIP phone is sending a SIP packet that Kamailio does not like.

We have a core dump file but I cannot read anything usefull from the backtrace. Can you see what is wrong from the backtrace?

Regards
Morten

The output from gdb (bt full):

Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio/kamailio.pid -m 256 -M 8 -u kamai'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f0e10de17b2 in cancel_branch (t=0x7f0dfbf38e10, branch=0, reason=<value optimized out>, flags=4) at t_cancel.c:284
284             if (cfg_get(tm, tm_cfg, reparse_invite) ||
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.107.el6.x86_64 hiredis-0.10.1-3.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6.x86_64 libcom_err-1.41.12-14.el6.x86_64 libselinux-2.0.94-5.3.el6.x86_64 libxml2-2.7.6-12.el6_4.1.x86_64 mysql-libs-5.1.67-1.el6_3.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64 openssl-1.0.0-27.el6_4.2.x86_64 zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0  0x00007f0e10de17b2 in cancel_branch (t=0x7f0dfbf38e10, branch=0, reason=<value optimized out>, flags=4) at t_cancel.c:284
        cancel = <value optimized out>
        len = <value optimized out>
        crb = 0x7f0dfbf39008
        irb = 0x7f0dfbf38f80
        ret = 1
        tmp_cd = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 0}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 0}}}}
        pcbuf = <value optimized out>
        __FUNCTION__ = "cancel_branch"
#1  0x00007f0e10e298ab in reply_received (p_msg=0x7f0e124ce760) at t_reply.c:2194
        msg_status = <value optimized out>
        last_uac_status = 408
        ack = 0x7f0dfbf38e10 "Ð
                               \267\373\r\177"
        ack_len = <value optimized out>
        branch = 0
        reply_status = <value optimized out>
        onreply_route = <value optimized out>
        cancel_data = {cancel_bitmap = 0, reason = {cause = 408, u = {text = {s = 0x0, len = 307468800}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 307468800}}}}
        uac = <value optimized out>
        t = 0x7f0dfbf38e10
        lack_dst = {send_sock = 0x0, to = {s = {sa_family = 6704, sa_data = "\\#\377\177\000\000\000\000\000\000\000\000\000"}, sin = {sin_family = 6704, sin_port = 9052, sin_addr = {s_addr = 32767},
              sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 6704, sin6_port = 9052, sin6_flowinfo = 32767, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\000\000\000\000\000\000\000\000p{1\022\016\177\000", __u6_addr16 = {0, 0, 0, 0, 31600, 4657, 32526, 0}, __u6_addr32 = {0, 0, 305232752, 32526}}},
              sin6_scope_id = 307029856}}, id = 32526, proto = 28 '\034', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        backup_user_from = <value optimized out>
        backup_user_to = <value optimized out>
        backup_domain_from = <value optimized out>
        backup_domain_to = <value optimized out>
        backup_uri_from = <value optimized out>
        backup_uri_to = <value optimized out>
        backup_xavps = <value optimized out>
        replies_locked = 0
        branch_ret = <value optimized out>
        prev_branch = <value optimized out>
        blst_503_timeout = <value optimized out>
        hf = <value optimized out>
        onsend_params = {req = 0x8d8a39, rpl = 0x541db4, param = 0x7f0e1253e528, code = 307029856, flags = 32526, branch = 0, t_rbuf = 0x7fff235c1a30, dst = 0x7f0e12317b70, send_buf = {
            s = 0x375311000000000 <Address 0x375311000000000 out of bounds>, len = 0}}
        ctx = {rec_lev = 307491008, run_flags = 32526, last_retcode = 5674412, jmp_env = {{__jmpbuf = {140733786626256, 63331951475841423, 139698413054576, 139698413299552, 9276465, 139698039855608,
                -63254168797292145, 63332490682325391}, __mask_was_saved = 0, __saved_mask = {__val = {139698413734928, 0, 139698411522821, 1, 140733786626608, 6185835, 5972697, 8586176, 9275699,
                  69026945952, 3, 9276465, 9275673, 139698413738496, 9275961, 139698413760704}}}}}
        __FUNCTION__ = "reply_received"
#2  0x0000000000456444 in do_forward_reply (msg=0x7f0e124ce760, mode=<value optimized out>) at forward.c:799
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
            sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}},
              sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = <value optimized out>
        r = <value optimized out>
        s = <value optimized out>
        len = <value optimized out>
        __FUNCTION__ = "do_forward_reply"
#3  0x000000000049e15e in receive_msg (buf=<value optimized out>, len=313, rcv_info=0x7fff235c1cd0) at receive.c:270
        msg = 0x7f0e124ce760
        ctx = {rec_lev = 11, run_flags = 0, last_retcode = 206110737, jmp_env = {{__jmpbuf = {139698036884436, 11, 219309716216, 139698419720192, 140733786627520, 4294967295, 140733786627647, 1},
              __mask_was_saved = 8576456, __saved_mask = {__val = {0, 28, 16, 0, 219305533392, 1, 0, 139698411461552, 219309716216, 139698036884436, 139698413732672, 139698419717800, 139698413732680,
                  140733786627416, 219305559701, 140733786627288}}}}}
        ret = <value optimized out>
        inb = {
          s = 0x8d8900 "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bK8149.c6575a95.0\r\nTo: sip:201@78799865.pbx.one-connect.dk;tag=07c44e68\r\nFrom: sip:201@78799865.pbx.one-connect.dk;tag=a6a1c5f60faecf035a"..., len = 313}
        __FUNCTION__ = "receive_msg"
#4  0x0000000000530e46 in udp_rcv_loop () at udp_server.c:557
---Type <return> to continue, or q <return> to quit---
        len = 313
        buf = "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bK8149.c6575a95.0\r\nTo: sip:201@78799865.pbx.one-connect.dk;tag=07c44e68\r\nFrom: sip:201@78799865.pbx.one-connect.dk;tag=a6a1c5f60faecf035a"...
        from = 0x7f0e12538340
        fromlen = 16
        ri = {src_ip = {af = 2, len = 4, u = {addrl = {2993962576, 0}, addr32 = {2993962576, 0, 0, 0}, addr16 = {15952, 45684, 0, 0, 0, 0, 0, 0}, addr = "P>t\262", '\000' <repeats 11 times>}}, dst_ip = {
            af = 2, len = 4, u = {addrl = {351868338, 0}, addr32 = {351868338, 0, 0, 0}, addr16 = {5554, 5369, 0, 0, 0, 0, 0, 0}, addr = "\262\025\371\024", '\000' <repeats 11 times>}}, src_port = 35754,
          dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\213\252P>t\262\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 43659,
              sin_addr = {s_addr = 2993962576}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 43659, sin6_flowinfo = 2993962576, sin6_addr = {__in6_u = {
                  __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x7f0e124cfbd0, proto = 1 '\001'}
        __FUNCTION__ = "udp_rcv_loop"
#5  0x000000000046716a in main_loop () at main.c:1638
        i = <value optimized out>
        pid = <value optimized out>
        si = <value optimized out>
        si_desc = "udp receiver child=2 sock=178.21.249.20:5060\000\000\000\000\200\303P\022\016\177\000\000\000\000\000\000\000\000\000\000\003\000\000\000\000\000\000\000\001\000\000\000\001\000\000\000@\350\216\000\000\000\000\000\001\000\000\000\000\000\000\000\200\350\216\000\000\000\000\000\000\000\200\020", '\000' <repeats 12 times>, "\005\000\000\000\000\000\000"
        nrprocs = <value optimized out>
        __FUNCTION__ = "main_loop"
#6  0x000000000046a002 in main (argc=<value optimized out>, argv=<value optimized out>) at main.c:2566
        cfg_stream = <value optimized out>
        c = <value optimized out>
        r = <value optimized out>
        tmp = 0x7fff235c377f ""
        tmp_len = 0
        options = 0x5c08c8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1722854551
        rfd = <value optimized out>
        debug_save = <value optimized out>
        debug_flag = <value optimized out>
        dont_fork_cnt = <value optimized out>
        n_lst = <value optimized out>
        p = <value optimized out>
        __FUNCTION__ = "main"
(gdb)





--
Morten Isaksen


_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users


-- 
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Kamailio Advanced Trainings - Berlin, Nov 25-28; Miami, Nov 18-20, 2013
  - more details about Kamailio trainings at http://www.asipto.com -

_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users




--
Morten Isaksen


_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users


-- 
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Kamailio Advanced Trainings - Berlin, Nov 25-28
  - more details about Kamailio trainings at http://www.asipto.com -

_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users




--
Morten Isaksen