1. # systemctl start ceph-osd@193
  2. ...
  3. cat ceph-osd.193.log
  4. 2018-07-05 13:35:08.606 7fe539e861c0  0 set uid:gid to 167:167 (ceph:ceph)
  5. 2018-07-05 13:35:08.606 7fe539e861c0  0 ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable), process (unknown), pid 480842
  6. 2018-07-05 13:35:08.606 7fe539e861c0  0 pidfile_write: ignore empty --pid-file
  7. 2018-07-05 13:35:08.693 7fe539e861c0  0 load: jerasure load: lrc load: isa 
  8. 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  9. 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  10. 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  11. 2018-07-05 13:35:08.694 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
  12. 2018-07-05 13:35:08.694 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
  13. 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) close
  14. 2018-07-05 13:35:08.954 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _mount path /var/lib/ceph/osd/ceph-193
  15. 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  16. 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  17. 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  18. 2018-07-05 13:35:08.955 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
  19. 2018-07-05 13:35:08.955 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
  20. 2018-07-05 13:35:08.957 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  21. 2018-07-05 13:35:08.957 7fe539e861c0  1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  22. 2018-07-05 13:35:08.958 7fe539e861c0  1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  23. 2018-07-05 13:35:08.958 7fe539e861c0  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-193/block size 7.3 TiB
  24. 2018-07-05 13:35:08.958 7fe539e861c0  1 bluefs mount
  25. 2018-07-05 13:35:09.281 7fe539e861c0 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_replay(bool, bool)' thread 7fe539e861c0 time 2018-07-05 13:35:09.278841
  26. /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: 828: FAILED assert(r != q->second->file_map.end())
  27.  
  28.  ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
  29.  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xff) [0x7fe5312e253f]
  30.  2: (()+0x286727) [0x7fe5312e2727]
  31.  3: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
  32.  4: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
  33.  5: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
  34.  6: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
  35.  7: (OSD::init()+0x3ea) [0x55eaee73476a]
  36.  8: (main()+0x226d) [0x55eaee62033d]
  37.  9: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
  38.  10: (()+0x37c600) [0x55eaee6f5600]
  39.  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  40.  
  41. --- begin dump of recent events ---
  42.   -140> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perfcounters_dump hook 0x55eaf072a320
  43.   -139> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command 1 hook 0x55eaf072a320
  44.   -138> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perf dump hook 0x55eaf072a320
  45.   -137> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perfcounters_schema hook 0x55eaf072a320
  46.   -136> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perf histogram dump hook 0x55eaf072a320
  47.   -135> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command 2 hook 0x55eaf072a320
  48.   -134> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perf schema hook 0x55eaf072a320
  49.   -133> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perf histogram schema hook 0x55eaf072a320
  50.   -132> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command perf reset hook 0x55eaf072a320
  51.   -131> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config show hook 0x55eaf072a320
  52.   -130> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config help hook 0x55eaf072a320
  53.   -129> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config set hook 0x55eaf072a320
  54.   -128> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config unset hook 0x55eaf072a320
  55.   -127> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config get hook 0x55eaf072a320
  56.   -126> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config diff hook 0x55eaf072a320
  57.   -125> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command config diff get hook 0x55eaf072a320
  58.   -124> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command log flush hook 0x55eaf072a320
  59.   -123> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command log dump hook 0x55eaf072a320
  60.   -122> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command log reopen hook 0x55eaf072a320
  61.   -121> 2018-07-05 13:35:08.559 7fe539e861c0  5 asok(0x55eaf07525a0) register_command dump_mempools hook 0x55eaf07602c8
  62.   -120> 2018-07-05 13:35:08.565 7fe539e861c0 10 monclient: get_monmap_and_config
  63.   -119> 2018-07-05 13:35:08.593 7fe539e861c0 10 monclient: build_initial_monmap
  64.   -118> 2018-07-05 13:35:08.595 7fe529fef700  2 Event(0x55eaf0739c80 nevent=5000 time_id=1).set_owner idx=0 owner=140622228813568
  65.   -117> 2018-07-05 13:35:08.595 7fe528fed700  2 Event(0x55eaf0a7ba80 nevent=5000 time_id=1).set_owner idx=2 owner=140622212028160
  66.   -116> 2018-07-05 13:35:08.595 7fe5297ee700  2 Event(0x55eaf0a7ae80 nevent=5000 time_id=1).set_owner idx=1 owner=140622220420864
  67.   -115> 2018-07-05 13:35:08.595 7fe539e861c0  1  Processor -- start
  68.   -114> 2018-07-05 13:35:08.595 7fe539e861c0  1 -- - start start
  69.   -113> 2018-07-05 13:35:08.595 7fe539e861c0 10 monclient: init
  70.   -112> 2018-07-05 13:35:08.595 7fe539e861c0  5 adding auth protocol: cephx
  71.   -111> 2018-07-05 13:35:08.595 7fe539e861c0 10 monclient: auth_supported 2 method cephx
  72.   -110> 2018-07-05 13:35:08.596 7fe539e861c0  2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-193/keyring
  73.   -109> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient: _reopen_session rank -1
  74.   -108> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): picked mon.noname-a con 0x55eaf0b4c600 addr 10.1.0.86:6789/0
  75.   -107> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): picked mon.noname-c con 0x55eaf0b4cc00 addr 10.1.1.37:6789/0
  76.   -106> 2018-07-05 13:35:08.596 7fe539e861c0  1 -- - --> 10.1.0.86:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- 0x55eaf0725200 con 0
  77.   -105> 2018-07-05 13:35:08.596 7fe539e861c0  1 -- - --> 10.1.1.37:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- 0x55eaf0725440 con 0
  78.   -104> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): _renew_subs
  79.   -103> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): authenticate will time out at 2018-07-05 13:40:08.596928
  80.   -102> 2018-07-05 13:35:08.597 7fe528fed700  1 -- 10.1.1.54:0/3032494579 learned_addr learned my addr 10.1.1.54:0/3032494579
  81.   -101> 2018-07-05 13:35:08.597 7fe528fed700  2 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
  82.   -100> 2018-07-05 13:35:08.597 7fe5297ee700  2 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
  83.    -99> 2018-07-05 13:35:08.597 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 1 0x55eaf0725440 mon_map magic: 0 v1
  84.    -98> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 1 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0725440 con 0x55eaf0b4cc00
  85.    -97> 2018-07-05 13:35:08.597 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 2 0x55eaf0725b00 auth_reply(proto 2 0 (0) Success) v1
  86.    -96> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): handle_monmap mon_map magic: 0 v1
  87.    -95> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting):  got monmap 2, mon.noname-c is now rank -1
  88.    -94> 2018-07-05 13:35:08.597 7fe5297ee700  5 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=274105 cs=1 l=1). rx mon.1 seq 1 0x55eaf0725200 mon_map magic: 0 v1
  89.    -93> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): dump:
  90. epoch 2
  91. fsid f035ee98-abfd-4496-b903-a403b29c828f
  92. last_changed 2018-06-20 16:49:45.273727
  93. created 2017-10-19 12:09:30.752686
  94. 0: 10.1.0.80:6789/0 mon.ceph2
  95. 1: 10.1.0.86:6789/0 mon.ceph1
  96. 2: 10.1.1.37:6789/0 mon.ceph3
  97.  
  98.    -92> 2018-07-05 13:35:08.597 7fe5297ee700  5 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=274105 cs=1 l=1). rx mon.1 seq 2 0x55eaf0b64000 auth_reply(proto 2 0 (0) Success) v1
  99.    -91> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2319497851 0 0) 0x55eaf0725b00 con 0x55eaf0b4cc00
  100.    -90> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): my global_id is 32888837
  101.    -89> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x55eaf0725440 con 0
  102.    -88> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.1 10.1.0.86:6789/0 1 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0725200 con 0x55eaf0b4c600
  103.    -87> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): handle_monmap mon_map magic: 0 v1
  104.    -86> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting):  got monmap 2, mon.ceph1 is now rank 1
  105.    -85> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): dump:
  106. epoch 2
  107. fsid f035ee98-abfd-4496-b903-a403b29c828f
  108. last_changed 2018-06-20 16:49:45.273727
  109. created 2017-10-19 12:09:30.752686
  110. 0: 10.1.0.80:6789/0 mon.ceph2
  111. 1: 10.1.0.86:6789/0 mon.ceph1
  112. 2: 10.1.1.37:6789/0 mon.ceph3
  113.  
  114.    -84> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.1 10.1.0.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2340670948 0 0) 0x55eaf0b64000 con 0x55eaf0b4c600
  115.    -83> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): my global_id is 32878462
  116.    -82> 2018-07-05 13:35:08.597 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 --> 10.1.0.86:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x55eaf0725b00 con 0
  117.    -81> 2018-07-05 13:35:08.598 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 3 0x55eaf0725440 auth_reply(proto 2 0 (0) Success) v1
  118.    -80> 2018-07-05 13:35:08.598 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (400396705 0 0) 0x55eaf0725440 con 0x55eaf0b4cc00
  119.    -79> 2018-07-05 13:35:08.598 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN pgs=274105 cs=1 l=1).mark_down
  120.    -78> 2018-07-05 13:35:08.598 7fe5287ec700  2 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN pgs=274105 cs=1 l=1)._stop
  121.    -77> 2018-07-05 13:35:08.598 7fe5287ec700  1 monclient: found mon.ceph3
  122.    -76> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _send_mon_message to mon.ceph3 at 10.1.1.37:6789/0
  123.    -75> 2018-07-05 13:35:08.598 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eaf0a7be00 con 0
  124.    -74> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2018-07-05 13:34:38.598631)
  125.    -73> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _send_mon_message to mon.ceph3 at 10.1.1.37:6789/0
  126.    -72> 2018-07-05 13:35:08.598 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- 0x55eaf0b64000 con 0
  127.    -71> 2018-07-05 13:35:08.598 7fe539e861c0  5 monclient: authenticate success, global_id 32888837
  128.    -70> 2018-07-05 13:35:08.603 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 4 0x55eaf0b58200 config(1 keys) v1
  129.    -69> 2018-07-05 13:35:08.603 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 5 0x55eaf0b64000 mon_map magic: 0 v1
  130.    -68> 2018-07-05 13:35:08.603 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 4 ==== config(1 keys) v1 ==== 41+0+0 (2911655278 0 0) 0x55eaf0b58200 con 0x55eaf0b4cc00
  131.    -67> 2018-07-05 13:35:08.603 7fe528fed700  5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 6 0x55eaf0725d40 auth_reply(proto 2 0 (0) Success) v1
  132.    -66> 2018-07-05 13:35:08.603 7fe5287ec700 10 monclient: handle_config config(1 keys) v1
  133.    -65> 2018-07-05 13:35:08.603 7fe539e861c0 10 monclient: get_monmap_and_config success
  134.    -64> 2018-07-05 13:35:08.603 7fe539e861c0 10 monclient: shutdown
  135.    -63> 2018-07-05 13:35:08.603 7fe526fe9700  4 set_mon_vals no callback set
  136.    -62> 2018-07-05 13:35:08.603 7fe526fe9700  4 set_mon_vals callback ignored osd_scrub_during_recovery
  137.    -61> 2018-07-05 13:35:08.603 7fe539e861c0  1 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN pgs=276670 cs=1 l=1).mark_down
  138.    -60> 2018-07-05 13:35:08.603 7fe5287ec700  1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 5 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0b64000 con 0x55eaf0b4cc00
  139.    -59> 2018-07-05 13:35:08.603 7fe539e861c0  2 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN pgs=276670 cs=1 l=1)._stop
  140.    -58> 2018-07-05 13:35:08.603 7fe526fe9700 10 set_mon_vals osd_scrub_during_recovery = true
  141.    -57> 2018-07-05 13:35:08.603 7fe5287ec700 10 monclient: discarding stray monitor message mon_map magic: 0 v1
  142.    -56> 2018-07-05 13:35:08.604 7fe539e861c0  1 -- 10.1.1.54:0/3032494579 shutdown_connections 
  143.    -55> 2018-07-05 13:35:08.604 7fe539e861c0  5 -- 10.1.1.54:0/3032494579 shutdown_connections mark down 10.1.1.37:6789/0 0x55eaf0b4cc00
  144.    -54> 2018-07-05 13:35:08.604 7fe539e861c0  5 -- 10.1.1.54:0/3032494579 shutdown_connections mark down 10.1.0.86:6789/0 0x55eaf0b4c600
  145.    -53> 2018-07-05 13:35:08.604 7fe539e861c0  5 -- 10.1.1.54:0/3032494579 shutdown_connections delete 0x55eaf0b4c600
  146.    -52> 2018-07-05 13:35:08.604 7fe539e861c0  5 -- 10.1.1.54:0/3032494579 shutdown_connections delete 0x55eaf0b4cc00
  147.    -51> 2018-07-05 13:35:08.604 7fe539e861c0  1 -- 10.1.1.54:0/3032494579 shutdown_connections 
  148.    -50> 2018-07-05 13:35:08.604 7fe539e861c0  1 -- 10.1.1.54:0/3032494579 wait complete.
  149.    -49> 2018-07-05 13:35:08.604 7fe539e861c0  1 -- 10.1.1.54:0/3032494579 >> 10.1.1.54:0/3032494579 conn(0x55eaf0b4c000 :-1 s=STATE_NONE pgs=0 cs=0 l=0).mark_down
  150.    -48> 2018-07-05 13:35:08.604 7fe539e861c0  2 -- 10.1.1.54:0/3032494579 >> 10.1.1.54:0/3032494579 conn(0x55eaf0b4c000 :-1 s=STATE_NONE pgs=0 cs=0 l=0)._stop
  151.    -47> 2018-07-05 13:35:08.606 7fe539e861c0  0 set uid:gid to 167:167 (ceph:ceph)
  152.    -46> 2018-07-05 13:35:08.606 7fe539e861c0  0 ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable), process (unknown), pid 480842
  153.    -45> 2018-07-05 13:35:08.606 7fe539e861c0  0 pidfile_write: ignore empty --pid-file
  154.    -44> 2018-07-05 13:35:08.637 7fe539e861c0  5 asok(0x55eaf07525a0) init /var/run/ceph/ceph-osd.193.asok
  155.    -43> 2018-07-05 13:35:08.637 7fe539e861c0  5 asok(0x55eaf07525a0) bind_and_listen /var/run/ceph/ceph-osd.193.asok
  156.    -42> 2018-07-05 13:35:08.638 7fe539e861c0  5 asok(0x55eaf07525a0) register_command 0 hook 0x55eaf07294a8
  157.    -41> 2018-07-05 13:35:08.638 7fe539e861c0  5 asok(0x55eaf07525a0) register_command version hook 0x55eaf07294a8
  158.    -40> 2018-07-05 13:35:08.638 7fe539e861c0  5 asok(0x55eaf07525a0) register_command git_version hook 0x55eaf07294a8
  159.    -39> 2018-07-05 13:35:08.638 7fe539e861c0  5 asok(0x55eaf07525a0) register_command help hook 0x55eaf072a520
  160.    -38> 2018-07-05 13:35:08.638 7fe539e861c0  5 asok(0x55eaf07525a0) register_command get_command_descriptions hook 0x55eaf072a510
  161.    -37> 2018-07-05 13:35:08.638 7fe527feb700  5 asok(0x55eaf07525a0) entry start
  162.    -36> 2018-07-05 13:35:08.638 7fe539e861c0  5 object store type is bluestore
  163.    -35> 2018-07-05 13:35:08.649 7fe539e861c0  1 -- 0.0.0.0:6968/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6968/480842
  164.    -34> 2018-07-05 13:35:08.656 7fe539e861c0  1 -- 0.0.0.0:6969/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6969/480842
  165.    -33> 2018-07-05 13:35:08.663 7fe539e861c0  1 -- 0.0.0.0:6970/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6970/480842
  166.    -32> 2018-07-05 13:35:08.670 7fe539e861c0  1 -- 0.0.0.0:6971/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6971/480842
  167.    -31> 2018-07-05 13:35:08.670 7fe539e861c0 10 monclient: build_initial_monmap
  168.    -30> 2018-07-05 13:35:08.693 7fe539e861c0  0 load: jerasure load: lrc load: isa 
  169.    -29> 2018-07-05 13:35:08.694 7fe539e861c0  5 adding auth protocol: cephx
  170.    -28> 2018-07-05 13:35:08.694 7fe539e861c0  5 adding auth protocol: cephx
  171.    -27> 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  172.    -26> 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  173.    -25> 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  174.    -24> 2018-07-05 13:35:08.694 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
  175.    -23> 2018-07-05 13:35:08.694 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
  176.    -22> 2018-07-05 13:35:08.694 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) close
  177.    -21> 2018-07-05 13:35:08.953 7fe539e861c0  5 asok(0x55eaf07525a0) register_command objecter_requests hook 0x55eaf072a6b0
  178.    -20> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- 0.0.0.0:6968/480842 start start
  179.    -19> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- - start start
  180.    -18> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- - start start
  181.    -17> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- 0.0.0.0:6971/480842 start start
  182.    -16> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- 0.0.0.0:6970/480842 start start
  183.    -15> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- 0.0.0.0:6969/480842 start start
  184.    -14> 2018-07-05 13:35:08.953 7fe539e861c0  1 -- - start start
  185.    -13> 2018-07-05 13:35:08.954 7fe539e861c0  2 osd.193 0 init /var/lib/ceph/osd/ceph-193 (looks like hdd)
  186.    -12> 2018-07-05 13:35:08.954 7fe539e861c0  2 osd.193 0 journal /var/lib/ceph/osd/ceph-193/journal
  187.    -11> 2018-07-05 13:35:08.954 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _mount path /var/lib/ceph/osd/ceph-193
  188.    -10> 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  189.     -9> 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  190.     -8> 2018-07-05 13:35:08.955 7fe539e861c0  1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  191.     -7> 2018-07-05 13:35:08.955 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
  192.     -6> 2018-07-05 13:35:08.955 7fe539e861c0  1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
  193.     -5> 2018-07-05 13:35:08.957 7fe539e861c0  1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
  194.     -4> 2018-07-05 13:35:08.957 7fe539e861c0  1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
  195.     -3> 2018-07-05 13:35:08.958 7fe539e861c0  1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
  196.     -2> 2018-07-05 13:35:08.958 7fe539e861c0  1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-193/block size 7.3 TiB
  197.     -1> 2018-07-05 13:35:08.958 7fe539e861c0  1 bluefs mount
  198.      0> 2018-07-05 13:35:09.281 7fe539e861c0 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_replay(bool, bool)' thread 7fe539e861c0 time 2018-07-05 13:35:09.278841
  199. /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: 828: FAILED assert(r != q->second->file_map.end())
  200.  
  201.  ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
  202.  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xff) [0x7fe5312e253f]
  203.  2: (()+0x286727) [0x7fe5312e2727]
  204.  3: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
  205.  4: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
  206.  5: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
  207.  6: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
  208.  7: (OSD::init()+0x3ea) [0x55eaee73476a]
  209.  8: (main()+0x226d) [0x55eaee62033d]
  210.  9: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
  211.  10: (()+0x37c600) [0x55eaee6f5600]
  212.  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  213.  
  214. --- logging levels ---
  215.    0/ 5 none
  216.    0/ 1 lockdep
  217.    0/ 1 context
  218.    1/ 1 crush
  219.    1/ 5 mds
  220.    1/ 5 mds_balancer
  221.    1/ 5 mds_locker
  222.    1/ 5 mds_log
  223.    1/ 5 mds_log_expire
  224.    1/ 5 mds_migrator
  225.    0/ 1 buffer
  226.    0/ 1 timer
  227.    0/ 1 filer
  228.    0/ 1 striper
  229.    0/ 1 objecter
  230.    0/ 5 rados
  231.    0/ 5 rbd
  232.    0/ 5 rbd_mirror
  233.    0/ 5 rbd_replay
  234.    0/ 5 journaler
  235.    0/ 5 objectcacher
  236.    0/ 5 client
  237.    1/ 5 osd
  238.    0/ 5 optracker
  239.    0/ 5 objclass
  240.    1/ 3 filestore
  241.    1/ 3 journal
  242.    0/ 5 ms
  243.    1/ 5 mon
  244.    0/10 monc
  245.    1/ 5 paxos
  246.    0/ 5 tp
  247.    1/ 5 auth
  248.    1/ 5 crypto
  249.    1/ 1 finisher
  250.    1/ 1 reserver
  251.    1/ 5 heartbeatmap
  252.    1/ 5 perfcounter
  253.    1/ 5 rgw
  254.    1/ 5 rgw_sync
  255.    1/10 civetweb
  256.    1/ 5 javaclient
  257.    1/ 5 asok
  258.    1/ 1 throttle
  259.    0/ 0 refs
  260.    1/ 5 xio
  261.    1/ 5 compressor
  262.    1/ 5 bluestore
  263.    1/ 5 bluefs
  264.    1/ 3 bdev
  265.    1/ 5 kstore
  266.    4/ 5 rocksdb
  267.    4/ 5 leveldb
  268.    4/ 5 memdb
  269.    1/ 5 kinetic
  270.    1/ 5 fuse
  271.    1/ 5 mgr
  272.    1/ 5 mgrc
  273.    1/ 5 dpdk
  274.    1/ 5 eventtrace
  275.   -2/-2 (syslog threshold)
  276.   -1/-1 (stderr threshold)
  277.   max_recent     10000
  278.   max_new         1000
  279.   log_file /var/log/ceph/ceph-osd.193.log
  280. --- end dump of recent events ---
  281. 2018-07-05 13:35:09.284 7fe539e861c0 -1 *** Caught signal (Aborted) **
  282.  in thread 7fe539e861c0 thread_name:ceph-osd
  283.  
  284.  ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
  285.  1: (()+0x8e1870) [0x55eaeec5a870]
  286.  2: (()+0xf680) [0x7fe52de43680]
  287.  3: (gsignal()+0x37) [0x7fe52ce64207]
  288.  4: (abort()+0x148) [0x7fe52ce658f8]
  289.  5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x25d) [0x7fe5312e269d]
  290.  6: (()+0x286727) [0x7fe5312e2727]
  291.  7: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
  292.  8: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
  293.  9: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
  294.  10: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
  295.  11: (OSD::init()+0x3ea) [0x55eaee73476a]
  296.  12: (main()+0x226d) [0x55eaee62033d]
  297.  13: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
  298.  14: (()+0x37c600) [0x55eaee6f5600]
  299.  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  300.  
  301. --- begin dump of recent events ---
  302.      0> 2018-07-05 13:35:09.284 7fe539e861c0 -1 *** Caught signal (Aborted) **
  303.  in thread 7fe539e861c0 thread_name:ceph-osd
  304.  
  305.  ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
  306.  1: (()+0x8e1870) [0x55eaeec5a870]
  307.  2: (()+0xf680) [0x7fe52de43680]
  308.  3: (gsignal()+0x37) [0x7fe52ce64207]
  309.  4: (abort()+0x148) [0x7fe52ce658f8]
  310.  5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x25d) [0x7fe5312e269d]
  311.  6: (()+0x286727) [0x7fe5312e2727]
  312.  7: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
  313.  8: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
  314.  9: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
  315.  10: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
  316.  11: (OSD::init()+0x3ea) [0x55eaee73476a]
  317.  12: (main()+0x226d) [0x55eaee62033d]
  318.  13: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
  319.  14: (()+0x37c600) [0x55eaee6f5600]
  320.  NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  321.  
  322. --- logging levels ---
  323.    0/ 5 none
  324.    0/ 1 lockdep
  325.    0/ 1 context
  326.    1/ 1 crush
  327.    1/ 5 mds
  328.    1/ 5 mds_balancer
  329.    1/ 5 mds_locker
  330.    1/ 5 mds_log
  331.    1/ 5 mds_log_expire
  332.    1/ 5 mds_migrator
  333.    0/ 1 buffer
  334.    0/ 1 timer
  335.    0/ 1 filer
  336.    0/ 1 striper
  337.    0/ 1 objecter
  338.    0/ 5 rados
  339.    0/ 5 rbd
  340.    0/ 5 rbd_mirror
  341.    0/ 5 rbd_replay
  342.    0/ 5 journaler
  343.    0/ 5 objectcacher
  344.    0/ 5 client
  345.    1/ 5 osd
  346.    0/ 5 optracker
  347.    0/ 5 objclass
  348.    1/ 3 filestore
  349.    1/ 3 journal
  350.    0/ 5 ms
  351.    1/ 5 mon
  352.    0/10 monc
  353.    1/ 5 paxos
  354.    0/ 5 tp
  355.    1/ 5 auth
  356.    1/ 5 crypto
  357.    1/ 1 finisher
  358.    1/ 1 reserver
  359.    1/ 5 heartbeatmap
  360.    1/ 5 perfcounter
  361.    1/ 5 rgw
  362.    1/ 5 rgw_sync
  363.    1/10 civetweb
  364.    1/ 5 javaclient
  365.    1/ 5 asok
  366.    1/ 1 throttle
  367.    0/ 0 refs
  368.    1/ 5 xio
  369.    1/ 5 compressor
  370.    1/ 5 bluestore
  371.    1/ 5 bluefs
  372.    1/ 3 bdev
  373.    1/ 5 kstore
  374.    4/ 5 rocksdb
  375.    4/ 5 leveldb
  376.    4/ 5 memdb
  377.    1/ 5 kinetic
  378.    1/ 5 fuse
  379.    1/ 5 mgr
  380.    1/ 5 mgrc
  381.    1/ 5 dpdk
  382.    1/ 5 eventtrace
  383.   -2/-2 (syslog threshold)
  384.   -1/-1 (stderr threshold)
  385.   max_recent     10000
  386.   max_new         1000
  387.   log_file /var/log/ceph/ceph-osd.193.log
  388. --- end dump of recent events ---