Squid: OSD crash

Bug #2107100 reported by Ponnuvel Palaniyappan
10
This bug affects 1 person
Affects Status Importance Assigned to Milestone
ceph (Ubuntu)
New
Undecided
Unassigned

Bug Description

Squid deployed OSDs crash when ESBs feature is enabled.

Backtrace:
```
{
    "assert_condition": "diff <= bytes_per_au[pos]",
    "assert_file": "./src/os/bluestore/bluestore_types.cc",
    "assert_func": "bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)",
    "assert_line": 511,
    "assert_msg": "./src/os/bluestore/bluestore_types.cc: In function 'bool bluestore_blob_use_tracker_t::put(uint32_t, uint32_t, PExtentVector*)' thread 7cec64a00640 time 2025-02-19T09:18:47.707999+0000\n./src/os/bluestore/bluestore_types.cc: 511: FAILED ceph_assert(diff <= bytes_per_au[pos])\n",
    "assert_thread_name": "tp_osd_tp",
    "backtrace": [
        "/lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7cec8e042520]",
        "pthread_kill()",
        "raise()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x182) [0x5b001d464d7b]",
        "/usr/bin/ceph-osd(+0x621edd) [0x5b001d464edd]",
        "(bluestore_blob_use_tracker_t::put(unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0x3e2) [0x5b001db4baa2]",
        "(BlueStore::Blob::put_ref(BlueStore::Collection*, unsigned int, unsigned int, std::vector<bluestore_pextent_t, mempool::pool_allocator<(mempool::pool_index_t)5, bluestore_pextent_t> >*)+0xa5) [0x5b001da86355]",
        "(BlueStore::OldExtent::create(boost::intrusive_ptr<BlueStore::Collection>, unsigned int, unsigned int, unsigned int, boost::intrusive_ptr<BlueStore::Blob>&)+0x11d) [0x5b001da866cd]",
        "(BlueStore::ExtentMap::punch_hole(boost::intrusive_ptr<BlueStore::Collection>&, unsigned long, unsigned long, boost::intrusive::list<BlueStore::OldExtent, boost::intrusive::member_hook<BlueStore::OldExtent, boost::intrusive::list_member_hook<>, &BlueStore::OldExtent::old_extent_item> >*)+0x3ee) [0x5b001da86c6e]",
        "(BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x2fe) [0x5b001dae6bce]",
        "(BlueStore::_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&, unsigned long)+0x98) [0x5b001dae6f78]",
        "(BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x1401) [0x5b001daf3271]",
        "(BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x2d0) [0x5b001db023f0]",
        "(non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ceph::os::Transaction, std::allocator<ceph::os::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x53) [0x5b001d735553]",
        "(ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr<OpRequest>, ECSubWrite&, ZTracer::Trace const&, ECListener&)+0xdd3) [0x5b001d9c9b73]",
        "(ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x670) [0x5b001d9d5900]",
        "(PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x49) [0x5b001d77a549]",
        "(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x4a9) [0x5b001d71e399]",
        "(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1b1) [0x5b001d571891]",
        "(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x69) [0x5b001d8908e9]",
        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x933) [0x5b001d58cde3]",
        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41b) [0x5b001dc270eb]",
        "(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x5b001dc29f74]",
        "/lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7cec8e094ac3]",
        "/lib/x86_64-linux-gnu/libc.so.6(+0x126850) [0x7cec8e126850]"
    ],
    "ceph_version": "19.2.0",
    "crash_id": "2025-02-19T09:18:47.722865Z_a4190ffa-00e2-4c19-8872-332d012ad6a9",
    "entity_name": "osd.124",
    "os_id": "22.04",
    "os_name": "Ubuntu 22.04.5 LTS",
    "os_version": "22.04.5 LTS (Jammy Jellyfish)",
    "os_version_id": "22.04",
    "process_name": "ceph-osd",
    "stack_sig": "cc5950be3a7269f6e9fd6489a4512f7ef2efafd437030fb72406144a171ca415",
    "timestamp": "2025-02-19T09:18:47.722865Z",
    "utsname_hostname": "lb5ap-cpst0014n",
    "utsname_machine": "x86_64",
    "utsname_release": "6.8.0-49-generic",
    "utsname_sysname": "Linux",
    "utsname_version": "#49~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Nov 6 17:42:15 UTC 2"
}

```

Ceph upstream tracker: https://tracker.ceph.com/issues/70390

This affects only Squid deployed OSDs (i.e. if the OSDs were originally deployed in an older releases and are now at Squid after upgrade, those aren't affected) and the only known workaround is to disable the feature with:

```
ceph config set osd bluestore_elastic_shared_blobs 0

```

and rebuild all the affected OSDs.

Tags: sts
tags: added: sts
To post a comment you must log in.
This report contains Public information  
Everyone can see this information.

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.