# systemctl start ceph-osd@193
...
cat ceph-osd.193.log
2018-07-05 13:35:08.606 7fe539e861c0 0 set uid:gid to 167:167 (ceph:ceph)
2018-07-05 13:35:08.606 7fe539e861c0 0 ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable), process (unknown), pid 480842
2018-07-05 13:35:08.606 7fe539e861c0 0 pidfile_write: ignore empty --pid-file
2018-07-05 13:35:08.693 7fe539e861c0 0 load: jerasure load: lrc load: isa
2018-07-05 13:35:08.694 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
2018-07-05 13:35:08.694 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
2018-07-05 13:35:08.694 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) close
2018-07-05 13:35:08.954 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _mount path /var/lib/ceph/osd/ceph-193
2018-07-05 13:35:08.955 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
2018-07-05 13:35:08.955 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
2018-07-05 13:35:08.955 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
2018-07-05 13:35:08.955 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
2018-07-05 13:35:08.955 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
2018-07-05 13:35:08.957 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
2018-07-05 13:35:08.957 7fe539e861c0 1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
2018-07-05 13:35:08.958 7fe539e861c0 1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
2018-07-05 13:35:08.958 7fe539e861c0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-193/block size 7.3 TiB
2018-07-05 13:35:08.958 7fe539e861c0 1 bluefs mount
2018-07-05 13:35:09.281 7fe539e861c0 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_replay(bool, bool)' thread 7fe539e861c0 time 2018-07-05 13:35:09.278841
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: 828: FAILED assert(r != q->second->file_map.end())
ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xff) [0x7fe5312e253f]
2: (()+0x286727) [0x7fe5312e2727]
3: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
4: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
5: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
6: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
7: (OSD::init()+0x3ea) [0x55eaee73476a]
8: (main()+0x226d) [0x55eaee62033d]
9: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
10: (()+0x37c600) [0x55eaee6f5600]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- begin dump of recent events ---
-140> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perfcounters_dump hook 0x55eaf072a320
-139> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command 1 hook 0x55eaf072a320
-138> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perf dump hook 0x55eaf072a320
-137> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perfcounters_schema hook 0x55eaf072a320
-136> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perf histogram dump hook 0x55eaf072a320
-135> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command 2 hook 0x55eaf072a320
-134> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perf schema hook 0x55eaf072a320
-133> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perf histogram schema hook 0x55eaf072a320
-132> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command perf reset hook 0x55eaf072a320
-131> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config show hook 0x55eaf072a320
-130> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config help hook 0x55eaf072a320
-129> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config set hook 0x55eaf072a320
-128> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config unset hook 0x55eaf072a320
-127> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config get hook 0x55eaf072a320
-126> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config diff hook 0x55eaf072a320
-125> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command config diff get hook 0x55eaf072a320
-124> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command log flush hook 0x55eaf072a320
-123> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command log dump hook 0x55eaf072a320
-122> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command log reopen hook 0x55eaf072a320
-121> 2018-07-05 13:35:08.559 7fe539e861c0 5 asok(0x55eaf07525a0) register_command dump_mempools hook 0x55eaf07602c8
-120> 2018-07-05 13:35:08.565 7fe539e861c0 10 monclient: get_monmap_and_config
-119> 2018-07-05 13:35:08.593 7fe539e861c0 10 monclient: build_initial_monmap
-118> 2018-07-05 13:35:08.595 7fe529fef700 2 Event(0x55eaf0739c80 nevent=5000 time_id=1).set_owner idx=0 owner=140622228813568
-117> 2018-07-05 13:35:08.595 7fe528fed700 2 Event(0x55eaf0a7ba80 nevent=5000 time_id=1).set_owner idx=2 owner=140622212028160
-116> 2018-07-05 13:35:08.595 7fe5297ee700 2 Event(0x55eaf0a7ae80 nevent=5000 time_id=1).set_owner idx=1 owner=140622220420864
-115> 2018-07-05 13:35:08.595 7fe539e861c0 1 Processor -- start
-114> 2018-07-05 13:35:08.595 7fe539e861c0 1 -- - start start
-113> 2018-07-05 13:35:08.595 7fe539e861c0 10 monclient: init
-112> 2018-07-05 13:35:08.595 7fe539e861c0 5 adding auth protocol: cephx
-111> 2018-07-05 13:35:08.595 7fe539e861c0 10 monclient: auth_supported 2 method cephx
-110> 2018-07-05 13:35:08.596 7fe539e861c0 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-193/keyring
-109> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient: _reopen_session rank -1
-108> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): picked mon.noname-a con 0x55eaf0b4c600 addr 10.1.0.86:6789/0
-107> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): picked mon.noname-c con 0x55eaf0b4cc00 addr 10.1.1.37:6789/0
-106> 2018-07-05 13:35:08.596 7fe539e861c0 1 -- - --> 10.1.0.86:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- 0x55eaf0725200 con 0
-105> 2018-07-05 13:35:08.596 7fe539e861c0 1 -- - --> 10.1.1.37:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- 0x55eaf0725440 con 0
-104> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): _renew_subs
-103> 2018-07-05 13:35:08.596 7fe539e861c0 10 monclient(hunting): authenticate will time out at 2018-07-05 13:40:08.596928
-102> 2018-07-05 13:35:08.597 7fe528fed700 1 -- 10.1.1.54:0/3032494579 learned_addr learned my addr 10.1.1.54:0/3032494579
-101> 2018-07-05 13:35:08.597 7fe528fed700 2 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
-100> 2018-07-05 13:35:08.597 7fe5297ee700 2 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_CONNECTING_WAIT_ACK_SEQ pgs=0 cs=0 l=0)._process_connection got newly_acked_seq 0 vs out_seq 0
-99> 2018-07-05 13:35:08.597 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 1 0x55eaf0725440 mon_map magic: 0 v1
-98> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 1 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0725440 con 0x55eaf0b4cc00
-97> 2018-07-05 13:35:08.597 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 2 0x55eaf0725b00 auth_reply(proto 2 0 (0) Success) v1
-96> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): handle_monmap mon_map magic: 0 v1
-95> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): got monmap 2, mon.noname-c is now rank -1
-94> 2018-07-05 13:35:08.597 7fe5297ee700 5 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=274105 cs=1 l=1). rx mon.1 seq 1 0x55eaf0725200 mon_map magic: 0 v1
-93> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): dump:
epoch 2
fsid f035ee98-abfd-4496-b903-a403b29c828f
last_changed 2018-06-20 16:49:45.273727
created 2017-10-19 12:09:30.752686
0: 10.1.0.80:6789/0 mon.ceph2
1: 10.1.0.86:6789/0 mon.ceph1
2: 10.1.1.37:6789/0 mon.ceph3
-92> 2018-07-05 13:35:08.597 7fe5297ee700 5 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=274105 cs=1 l=1). rx mon.1 seq 2 0x55eaf0b64000 auth_reply(proto 2 0 (0) Success) v1
-91> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2319497851 0 0) 0x55eaf0725b00 con 0x55eaf0b4cc00
-90> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): my global_id is 32888837
-89> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x55eaf0725440 con 0
-88> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.1 10.1.0.86:6789/0 1 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0725200 con 0x55eaf0b4c600
-87> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): handle_monmap mon_map magic: 0 v1
-86> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): got monmap 2, mon.ceph1 is now rank 1
-85> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): dump:
epoch 2
fsid f035ee98-abfd-4496-b903-a403b29c828f
last_changed 2018-06-20 16:49:45.273727
created 2017-10-19 12:09:30.752686
0: 10.1.0.80:6789/0 mon.ceph2
1: 10.1.0.86:6789/0 mon.ceph1
2: 10.1.1.37:6789/0 mon.ceph3
-84> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.1 10.1.0.86:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2340670948 0 0) 0x55eaf0b64000 con 0x55eaf0b4c600
-83> 2018-07-05 13:35:08.597 7fe5287ec700 10 monclient(hunting): my global_id is 32878462
-82> 2018-07-05 13:35:08.597 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 --> 10.1.0.86:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- 0x55eaf0725b00 con 0
-81> 2018-07-05 13:35:08.598 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 3 0x55eaf0725440 auth_reply(proto 2 0 (0) Success) v1
-80> 2018-07-05 13:35:08.598 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (400396705 0 0) 0x55eaf0725440 con 0x55eaf0b4cc00
-79> 2018-07-05 13:35:08.598 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN pgs=274105 cs=1 l=1).mark_down
-78> 2018-07-05 13:35:08.598 7fe5287ec700 2 -- 10.1.1.54:0/3032494579 >> 10.1.0.86:6789/0 conn(0x55eaf0b4c600 :-1 s=STATE_OPEN pgs=274105 cs=1 l=1)._stop
-77> 2018-07-05 13:35:08.598 7fe5287ec700 1 monclient: found mon.ceph3
-76> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _send_mon_message to mon.ceph3 at 10.1.1.37:6789/0
-75> 2018-07-05 13:35:08.598 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eaf0a7be00 con 0
-74> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2018-07-05 13:34:38.598631)
-73> 2018-07-05 13:35:08.598 7fe5287ec700 10 monclient: _send_mon_message to mon.ceph3 at 10.1.1.37:6789/0
-72> 2018-07-05 13:35:08.598 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 --> 10.1.1.37:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- 0x55eaf0b64000 con 0
-71> 2018-07-05 13:35:08.598 7fe539e861c0 5 monclient: authenticate success, global_id 32888837
-70> 2018-07-05 13:35:08.603 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 4 0x55eaf0b58200 config(1 keys) v1
-69> 2018-07-05 13:35:08.603 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 5 0x55eaf0b64000 mon_map magic: 0 v1
-68> 2018-07-05 13:35:08.603 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 4 ==== config(1 keys) v1 ==== 41+0+0 (2911655278 0 0) 0x55eaf0b58200 con 0x55eaf0b4cc00
-67> 2018-07-05 13:35:08.603 7fe528fed700 5 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=276670 cs=1 l=1). rx mon.2 seq 6 0x55eaf0725d40 auth_reply(proto 2 0 (0) Success) v1
-66> 2018-07-05 13:35:08.603 7fe5287ec700 10 monclient: handle_config config(1 keys) v1
-65> 2018-07-05 13:35:08.603 7fe539e861c0 10 monclient: get_monmap_and_config success
-64> 2018-07-05 13:35:08.603 7fe539e861c0 10 monclient: shutdown
-63> 2018-07-05 13:35:08.603 7fe526fe9700 4 set_mon_vals no callback set
-62> 2018-07-05 13:35:08.603 7fe526fe9700 4 set_mon_vals callback ignored osd_scrub_during_recovery
-61> 2018-07-05 13:35:08.603 7fe539e861c0 1 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN pgs=276670 cs=1 l=1).mark_down
-60> 2018-07-05 13:35:08.603 7fe5287ec700 1 -- 10.1.1.54:0/3032494579 <== mon.2 10.1.1.37:6789/0 5 ==== mon_map magic: 0 v1 ==== 397+0+0 (1434546537 0 0) 0x55eaf0b64000 con 0x55eaf0b4cc00
-59> 2018-07-05 13:35:08.603 7fe539e861c0 2 -- 10.1.1.54:0/3032494579 >> 10.1.1.37:6789/0 conn(0x55eaf0b4cc00 :-1 s=STATE_OPEN pgs=276670 cs=1 l=1)._stop
-58> 2018-07-05 13:35:08.603 7fe526fe9700 10 set_mon_vals osd_scrub_during_recovery = true
-57> 2018-07-05 13:35:08.603 7fe5287ec700 10 monclient: discarding stray monitor message mon_map magic: 0 v1
-56> 2018-07-05 13:35:08.604 7fe539e861c0 1 -- 10.1.1.54:0/3032494579 shutdown_connections
-55> 2018-07-05 13:35:08.604 7fe539e861c0 5 -- 10.1.1.54:0/3032494579 shutdown_connections mark down 10.1.1.37:6789/0 0x55eaf0b4cc00
-54> 2018-07-05 13:35:08.604 7fe539e861c0 5 -- 10.1.1.54:0/3032494579 shutdown_connections mark down 10.1.0.86:6789/0 0x55eaf0b4c600
-53> 2018-07-05 13:35:08.604 7fe539e861c0 5 -- 10.1.1.54:0/3032494579 shutdown_connections delete 0x55eaf0b4c600
-52> 2018-07-05 13:35:08.604 7fe539e861c0 5 -- 10.1.1.54:0/3032494579 shutdown_connections delete 0x55eaf0b4cc00
-51> 2018-07-05 13:35:08.604 7fe539e861c0 1 -- 10.1.1.54:0/3032494579 shutdown_connections
-50> 2018-07-05 13:35:08.604 7fe539e861c0 1 -- 10.1.1.54:0/3032494579 wait complete.
-49> 2018-07-05 13:35:08.604 7fe539e861c0 1 -- 10.1.1.54:0/3032494579 >> 10.1.1.54:0/3032494579 conn(0x55eaf0b4c000 :-1 s=STATE_NONE pgs=0 cs=0 l=0).mark_down
-48> 2018-07-05 13:35:08.604 7fe539e861c0 2 -- 10.1.1.54:0/3032494579 >> 10.1.1.54:0/3032494579 conn(0x55eaf0b4c000 :-1 s=STATE_NONE pgs=0 cs=0 l=0)._stop
-47> 2018-07-05 13:35:08.606 7fe539e861c0 0 set uid:gid to 167:167 (ceph:ceph)
-46> 2018-07-05 13:35:08.606 7fe539e861c0 0 ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable), process (unknown), pid 480842
-45> 2018-07-05 13:35:08.606 7fe539e861c0 0 pidfile_write: ignore empty --pid-file
-44> 2018-07-05 13:35:08.637 7fe539e861c0 5 asok(0x55eaf07525a0) init /var/run/ceph/ceph-osd.193.asok
-43> 2018-07-05 13:35:08.637 7fe539e861c0 5 asok(0x55eaf07525a0) bind_and_listen /var/run/ceph/ceph-osd.193.asok
-42> 2018-07-05 13:35:08.638 7fe539e861c0 5 asok(0x55eaf07525a0) register_command 0 hook 0x55eaf07294a8
-41> 2018-07-05 13:35:08.638 7fe539e861c0 5 asok(0x55eaf07525a0) register_command version hook 0x55eaf07294a8
-40> 2018-07-05 13:35:08.638 7fe539e861c0 5 asok(0x55eaf07525a0) register_command git_version hook 0x55eaf07294a8
-39> 2018-07-05 13:35:08.638 7fe539e861c0 5 asok(0x55eaf07525a0) register_command help hook 0x55eaf072a520
-38> 2018-07-05 13:35:08.638 7fe539e861c0 5 asok(0x55eaf07525a0) register_command get_command_descriptions hook 0x55eaf072a510
-37> 2018-07-05 13:35:08.638 7fe527feb700 5 asok(0x55eaf07525a0) entry start
-36> 2018-07-05 13:35:08.638 7fe539e861c0 5 object store type is bluestore
-35> 2018-07-05 13:35:08.649 7fe539e861c0 1 -- 0.0.0.0:6968/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6968/480842
-34> 2018-07-05 13:35:08.656 7fe539e861c0 1 -- 0.0.0.0:6969/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6969/480842
-33> 2018-07-05 13:35:08.663 7fe539e861c0 1 -- 0.0.0.0:6970/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6970/480842
-32> 2018-07-05 13:35:08.670 7fe539e861c0 1 -- 0.0.0.0:6971/480842 _finish_bind bind my_inst.addr is 0.0.0.0:6971/480842
-31> 2018-07-05 13:35:08.670 7fe539e861c0 10 monclient: build_initial_monmap
-30> 2018-07-05 13:35:08.693 7fe539e861c0 0 load: jerasure load: lrc load: isa
-29> 2018-07-05 13:35:08.694 7fe539e861c0 5 adding auth protocol: cephx
-28> 2018-07-05 13:35:08.694 7fe539e861c0 5 adding auth protocol: cephx
-27> 2018-07-05 13:35:08.694 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
-26> 2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
-25> 2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
-24> 2018-07-05 13:35:08.694 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
-23> 2018-07-05 13:35:08.694 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
-22> 2018-07-05 13:35:08.694 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) close
-21> 2018-07-05 13:35:08.953 7fe539e861c0 5 asok(0x55eaf07525a0) register_command objecter_requests hook 0x55eaf072a6b0
-20> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- 0.0.0.0:6968/480842 start start
-19> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- - start start
-18> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- - start start
-17> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- 0.0.0.0:6971/480842 start start
-16> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- 0.0.0.0:6970/480842 start start
-15> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- 0.0.0.0:6969/480842 start start
-14> 2018-07-05 13:35:08.953 7fe539e861c0 1 -- - start start
-13> 2018-07-05 13:35:08.954 7fe539e861c0 2 osd.193 0 init /var/lib/ceph/osd/ceph-193 (looks like hdd)
-12> 2018-07-05 13:35:08.954 7fe539e861c0 2 osd.193 0 journal /var/lib/ceph/osd/ceph-193/journal
-11> 2018-07-05 13:35:08.954 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _mount path /var/lib/ceph/osd/ceph-193
-10> 2018-07-05 13:35:08.955 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
-9> 2018-07-05 13:35:08.955 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
-8> 2018-07-05 13:35:08.955 7fe539e861c0 1 bdev(0x55eaf0ab6000 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
-7> 2018-07-05 13:35:08.955 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes kv_min_ratio 1 > kv_ratio 0.5
-6> 2018-07-05 13:35:08.955 7fe539e861c0 1 bluestore(/var/lib/ceph/osd/ceph-193) _set_cache_sizes cache_size 209715200 meta 0 kv 1 data 0
-5> 2018-07-05 13:35:08.957 7fe539e861c0 1 bdev create path /var/lib/ceph/osd/ceph-193/block type kernel
-4> 2018-07-05 13:35:08.957 7fe539e861c0 1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open path /var/lib/ceph/osd/ceph-193/block
-3> 2018-07-05 13:35:08.958 7fe539e861c0 1 bdev(0x55eaf0ab6a80 /var/lib/ceph/osd/ceph-193/block) open size 8001457295360 (0x746fc051000, 7.3 TiB) block_size 4096 (4 KiB) rotational
-2> 2018-07-05 13:35:08.958 7fe539e861c0 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-193/block size 7.3 TiB
-1> 2018-07-05 13:35:08.958 7fe539e861c0 1 bluefs mount
0> 2018-07-05 13:35:09.281 7fe539e861c0 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_replay(bool, bool)' thread 7fe539e861c0 time 2018-07-05 13:35:09.278841
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/13.2.0/rpm/el7/BUILD/ceph-13.2.0/src/os/bluestore/BlueFS.cc: 828: FAILED assert(r != q->second->file_map.end())
ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xff) [0x7fe5312e253f]
2: (()+0x286727) [0x7fe5312e2727]
3: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
4: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
5: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
6: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
7: (OSD::init()+0x3ea) [0x55eaee73476a]
8: (main()+0x226d) [0x55eaee62033d]
9: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
10: (()+0x37c600) [0x55eaee6f5600]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 kinetic
1/ 5 fuse
1/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-osd.193.log
--- end dump of recent events ---
2018-07-05 13:35:09.284 7fe539e861c0 -1 *** Caught signal (Aborted) **
in thread 7fe539e861c0 thread_name:ceph-osd
ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
1: (()+0x8e1870) [0x55eaeec5a870]
2: (()+0xf680) [0x7fe52de43680]
3: (gsignal()+0x37) [0x7fe52ce64207]
4: (abort()+0x148) [0x7fe52ce658f8]
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x25d) [0x7fe5312e269d]
6: (()+0x286727) [0x7fe5312e2727]
7: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
8: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
9: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
10: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
11: (OSD::init()+0x3ea) [0x55eaee73476a]
12: (main()+0x226d) [0x55eaee62033d]
13: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
14: (()+0x37c600) [0x55eaee6f5600]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- begin dump of recent events ---
0> 2018-07-05 13:35:09.284 7fe539e861c0 -1 *** Caught signal (Aborted) **
in thread 7fe539e861c0 thread_name:ceph-osd
ceph version 13.2.0 (79a10589f1f80dfe21e8f9794365ed98143071c4) mimic (stable)
1: (()+0x8e1870) [0x55eaeec5a870]
2: (()+0xf680) [0x7fe52de43680]
3: (gsignal()+0x37) [0x7fe52ce64207]
4: (abort()+0x148) [0x7fe52ce658f8]
5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x25d) [0x7fe5312e269d]
6: (()+0x286727) [0x7fe5312e2727]
7: (BlueFS::_replay(bool, bool)+0x4364) [0x55eaeec13e34]
8: (BlueFS::mount()+0x1d9) [0x55eaeec14219]
9: (BlueStore::_open_db(bool, bool)+0x15ae) [0x55eaeeb42c0e]
10: (BlueStore::_mount(bool, bool)+0x451) [0x55eaeeb6f361]
11: (OSD::init()+0x3ea) [0x55eaee73476a]
12: (main()+0x226d) [0x55eaee62033d]
13: (__libc_start_main()+0xf5) [0x7fe52ce503d5]
14: (()+0x37c600) [0x55eaee6f5600]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 5 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 kinetic
1/ 5 fuse
1/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
-2/-2 (syslog threshold)
-1/-1 (stderr threshold)
max_recent 10000
max_new 1000
log_file /var/log/ceph/ceph-osd.193.log
--- end dump of recent events ---