[Bug 2107100] [NEW] Squid: OSD crash
Ponnuvel Palaniyappan
2107100 at bugs.launchpad.net
Fri Apr 11 15:07:05 UTC 2025
Public bug reported:
Squid deployed OSDs crash when ESBs feature is enabled.
Backtrace:
```
{
"assert_condition": "diff <= bytes_per_au[pos]",
"assert_file": "./src/os/bluestore/bluestore_types.cc",
"assert_func": "bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)",
"assert_line": 511,
"assert_msg": "./src/os/bluestore/bluestore_types.cc: In function 'bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)' thread 7cec64a00640 time 2025-02-19T09:18:47.707999+0000\n./src/os/bluestore/bluestore_types.cc: 511: FAILED ceph_assert(diff <= bytes_per_au[pos])\n",
"assert_thread_name": "tp_osd_tp",
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7cec8e042520]",
"pthread_kill()",
"raise()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x182) [0x5b001d464d7b]",
"/usr/bin/ceph-osd(+0x621edd) [0x5b001d464edd]",
"(bluestore_blob_use_tracker_t::put(unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0x3e2) [0x5b001db4baa2]",
"(BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0xa5) [0x5b001da86355]",
"(BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>, unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x5b001da866cd]",
"(BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&, unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x3ee) [0x5b001da86c6e]",
"(BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x2fe) [0x5b001dae6bce]",
"(BlueStore::_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long)+0x98) [0x5b001dae6f78]",
"(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1401) [0x5b001daf3271]",
"(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2d0) [0x5b001db023f0]",
"(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x53) [0x5b001d735553]",
"(ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr<OpRequest>, ECSubWrite&, ZTracer::Trace const&, ECListener&)+0xdd3) [0x5b001d9c9b73]",
"(ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x670) [0x5b001d9d5900]",
"(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x49) [0x5b001d77a549]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x4a9) [0x5b001d71e399]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1b1) [0x5b001d571891]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x69) [0x5b001d8908e9]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x933) [0x5b001d58cde3]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41b) [0x5b001dc270eb]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5b001dc29f74]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7cec8e094ac3]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x126850) [0x7cec8e126850]"
],
"ceph_version": "19.2.0",
"crash_id": "2025-02-19T09:18:47.722865Z_a4190ffa-00e2-4c19-8872-332d012ad6a9",
"entity_name": "osd.124",
"os_id": "22.04",
"os_name": "Ubuntu 22.04.5 LTS",
"os_version": "22.04.5 LTS (Jammy Jellyfish)",
"os_version_id": "22.04",
"process_name": "ceph-osd",
"stack_sig": "cc5950be3a7269f6e9fd6489a4512f7ef2efafd437030fb72406144a171ca415",
"timestamp": "2025-02-19T09:18:47.722865Z",
"utsname_hostname": "lb5ap-cpst0014n",
"utsname_machine": "x86_64",
"utsname_release": "6.8.0-49-generic",
"utsname_sysname": "Linux",
"utsname_version": "#49~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Nov 6 17:42:15 UTC 2"
}
```
Ceph upstream tracker: https://tracker.ceph.com/issues/70390
This affects only Squid deployed OSDs (i.e. if the OSDs were originally
deployed in an older releases and are now at Squid after upgrade, those
aren't affected) and the only known workaround is to disable the feature
with:
```
ceph config set osd bluestore_elastic_shared_blobs 0
```
and rebuild all the affected OSDs.
** Affects: ceph (Ubuntu)
Importance: Undecided
Status: New
** Tags: sts
** Tags added: sts
--
You received this bug notification because you are a member of Ubuntu
OpenStack, which is subscribed to ceph in Ubuntu.
https://bugs.launchpad.net/bugs/2107100
Title:
Squid: OSD crash
Status in ceph package in Ubuntu:
New
Bug description:
Squid deployed OSDs crash when ESBs feature is enabled.
Backtrace:
```
{
"assert_condition": "diff <= bytes_per_au[pos]",
"assert_file": "./src/os/bluestore/bluestore_types.cc",
"assert_func": "bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)",
"assert_line": 511,
"assert_msg": "./src/os/bluestore/bluestore_types.cc: In function 'bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)' thread 7cec64a00640 time 2025-02-19T09:18:47.707999+0000\n./src/os/bluestore/bluestore_types.cc: 511: FAILED ceph_assert(diff <= bytes_per_au[pos])\n",
"assert_thread_name": "tp_osd_tp",
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7cec8e042520]",
"pthread_kill()",
"raise()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x182) [0x5b001d464d7b]",
"/usr/bin/ceph-osd(+0x621edd) [0x5b001d464edd]",
"(bluestore_blob_use_tracker_t::put(unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0x3e2) [0x5b001db4baa2]",
"(BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0xa5) [0x5b001da86355]",
"(BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>, unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x5b001da866cd]",
"(BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&, unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x3ee) [0x5b001da86c6e]",
"(BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x2fe) [0x5b001dae6bce]",
"(BlueStore::_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long)+0x98) [0x5b001dae6f78]",
"(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1401) [0x5b001daf3271]",
"(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2d0) [0x5b001db023f0]",
"(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x53) [0x5b001d735553]",
"(ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr<OpRequest>, ECSubWrite&, ZTracer::Trace const&, ECListener&)+0xdd3) [0x5b001d9c9b73]",
"(ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x670) [0x5b001d9d5900]",
"(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x49) [0x5b001d77a549]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x4a9) [0x5b001d71e399]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1b1) [0x5b001d571891]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x69) [0x5b001d8908e9]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x933) [0x5b001d58cde3]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41b) [0x5b001dc270eb]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5b001dc29f74]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7cec8e094ac3]",
"/lib/x86_64-linux-gnu/libc.so.6(+0x126850) [0x7cec8e126850]"
],
"ceph_version": "19.2.0",
"crash_id": "2025-02-19T09:18:47.722865Z_a4190ffa-00e2-4c19-8872-332d012ad6a9",
"entity_name": "osd.124",
"os_id": "22.04",
"os_name": "Ubuntu 22.04.5 LTS",
"os_version": "22.04.5 LTS (Jammy Jellyfish)",
"os_version_id": "22.04",
"process_name": "ceph-osd",
"stack_sig": "cc5950be3a7269f6e9fd6489a4512f7ef2efafd437030fb72406144a171ca415",
"timestamp": "2025-02-19T09:18:47.722865Z",
"utsname_hostname": "lb5ap-cpst0014n",
"utsname_machine": "x86_64",
"utsname_release": "6.8.0-49-generic",
"utsname_sysname": "Linux",
"utsname_version": "#49~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Nov 6 17:42:15 UTC 2"
}
```
Ceph upstream tracker: https://tracker.ceph.com/issues/70390
This affects only Squid deployed OSDs (i.e. if the OSDs were
originally deployed in an older releases and are now at Squid after
upgrade, those aren't affected) and the only known workaround is to
disable the feature with:
```
ceph config set osd bluestore_elastic_shared_blobs 0
```
and rebuild all the affected OSDs.
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/ceph/+bug/2107100/+subscriptions
More information about the Ubuntu-openstack-bugs
mailing list