LustreError: 156698:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff8af200a79000
LustreError: 156698:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 156698 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x125/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7fb566ca18fb

Lustre: 152447:0:(osd_internal.h:1335:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 253 < left 312, rollback = 2
Lustre: 152447:0:(osd_handler.c:1966:osd_trans_dump_creds()) create: 1/4/4, destroy: 0/0/0
Lustre: 152447:0:(osd_handler.c:1973:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 6/312/0
Lustre: 152447:0:(osd_handler.c:1983:osd_trans_dump_creds()) write: 3/39/0, punch: 0/0/0, quota 1/3/2
Lustre: 149514:0:(osd_handler.c:1990:osd_trans_dump_creds()) insert: 3/69/1, delete: 0/0/0
Lustre: 152447:0:(osd_handler.c:1983:osd_trans_dump_creds()) Skipped 1 previous similar message
Lustre: 149514:0:(osd_handler.c:1997:osd_trans_dump_creds()) ref_add: 1/1/0, ref_del: 0/0/0
Lustre: 149514:0:(osd_handler.c:1997:osd_trans_dump_creds()) Skipped 3 previous similar messages
Lustre: 149515:0:(osd_internal.h:1335:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 260 < left 387, rollback = 2
Lustre: 149515:0:(osd_internal.h:1335:osd_trans_exec_op()) Skipped 37 previous similar messages
Lustre: 149515:0:(osd_handler.c:1966:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 149515:0:(osd_handler.c:1966:osd_trans_dump_creds()) Skipped 37 previous similar messages
Lustre: 149515:0:(osd_handler.c:1973:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 5/387/0
Lustre: 149515:0:(osd_handler.c:1973:osd_trans_dump_creds()) Skipped 37 previous similar messages
Lustre: 149515:0:(osd_handler.c:1983:osd_trans_dump_creds()) write: 5/18/0, punch: 0/0/0, quota 1/3/0
Lustre: 149515:0:(osd_handler.c:1983:osd_trans_dump_creds()) Skipped 36 previous similar messages
Lustre: 149515:0:(osd_handler.c:1990:osd_trans_dump_creds()) insert: 1/36/0, delete: 2/2/0
Lustre: 149515:0:(osd_handler.c:1990:osd_trans_dump_creds()) Skipped 37 previous similar messages
Lustre: 149515:0:(osd_handler.c:1997:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 1/1/0
Lustre: 149515:0:(osd_handler.c:1997:osd_trans_dump_creds()) Skipped 34 previous similar messages
Lustre: 149515:0:(osd_internal.h:1335:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 260 < left 387, rollback = 2
Lustre: 149515:0:(osd_internal.h:1335:osd_trans_exec_op()) Skipped 119 previous similar messages
Lustre: 149515:0:(osd_handler.c:1966:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 149515:0:(osd_handler.c:1966:osd_trans_dump_creds()) Skipped 119 previous similar messages
Lustre: 149515:0:(osd_handler.c:1973:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 5/387/0
Lustre: 149515:0:(osd_handler.c:1973:osd_trans_dump_creds()) Skipped 119 previous similar messages
Lustre: 149515:0:(osd_handler.c:1983:osd_trans_dump_creds()) write: 5/21/0, punch: 0/0/0, quota 1/3/0
Lustre: 149515:0:(osd_handler.c:1983:osd_trans_dump_creds()) Skipped 119 previous similar messages
Lustre: 149515:0:(osd_handler.c:1990:osd_trans_dump_creds()) insert: 1/36/0, delete: 2/2/0
Lustre: 149515:0:(osd_handler.c:1990:osd_trans_dump_creds()) Skipped 119 previous similar messages
Lustre: 149515:0:(osd_handler.c:1997:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 1/1/0
Lustre: 149515:0:(osd_handler.c:1997:osd_trans_dump_creds()) Skipped 119 previous similar messages
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl lfsck_start -M lustre-MDT0000 -t namespace -A -r
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d -f /mnt/lustre-mds1
Lustre: lustre-MDT0000-osp-MDT0002: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
Lustre: Skipped 14 previous similar messages
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000009b68d18a[0x5, 1, [0x200003ab1:0x20:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000010d374f2mdt-object@000000009b68d18a( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000e4221d00mdd-object@00000000e4221d00(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000039ad533lod-object@00000000039ad533
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000035a51ff1osd-ldiskfs-object@0000000035a51ff1(i:000000000341bbfa:199/909937154)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000009b68d18a
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000256b072f[0x5, 1, [0x200003ab1:0x16:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000193abe8cmdt-object@00000000256b072f( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000c614ef70mdd-object@00000000c614ef70(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000004cbceda2lod-object@000000004cbceda2
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000005ae9477dosd-ldiskfs-object@000000005ae9477d(i:00000000284a10bc:189/1552446718)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000256b072f
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000ae1d004a[0x5, 1, [0x200003ab1:0x2b:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000662a046emdt-object@00000000ae1d004a( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000328198eemdd-object@00000000328198ee(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000002187ad47lod-object@000000002187ad47
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000e50bcb59osd-ldiskfs-object@00000000e50bcb59(i:00000000fb1117a4:210/1636272179)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000ae1d004a
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@0000000066b85116[0x5, 1, [0x200003ab1:0x19:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000005ea74f97mdt-object@0000000066b85116( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000004bef40fcmdd-object@000000004bef40fc(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000003eaaa02lod-object@0000000003eaaa02
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000138eb17bosd-ldiskfs-object@00000000138eb17b(i:0000000029cde013:192/583573381)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@0000000066b85116
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000d8dc89dc[0x5, 1, [0x200003ab1:0xf:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b1764b18mdt-object@00000000d8dc89dc( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000045031b03mdd-object@0000000045031b03(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000059015860lod-object@0000000059015860
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000083946a44osd-ldiskfs-object@0000000083946a44(i:00000000b10497ec:180/1829992641)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000d8dc89dc
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000e53ac637[0x5, 1, [0x200003ab1:0x12:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000044513458mdt-object@00000000e53ac637( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000074645d05mdd-object@0000000074645d05(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000ab7647eelod-object@00000000ab7647ee
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000a0225804osd-ldiskfs-object@00000000a0225804(i:00000000aa675d3f:185/944165849)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000e53ac637
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000edf9b37d[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000746ae26d
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000e02acc5aosd-ldiskfs-object@00000000e02acc5a(i:00000000402ec0e1:13/2035956162)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000edf9b37d
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000003b4a0a1b[0x5, 1, [0x200003ab1:0x27:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000674f8608mdt-object@000000003b4a0a1b( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000030934eb8mdd-object@0000000030934eb8(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000002d84f21lod-object@0000000002d84f21
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000a6eb8a3fosd-ldiskfs-object@00000000a6eb8a3f(i:000000007f8e4cd9:206/1868555257)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000003b4a0a1b
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000009005dc8c[0x5, 1, [0x200003ab1:0x15:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000008658eeb7mdt-object@000000009005dc8c( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000004a8096c8mdd-object@000000004a8096c8(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000b9cd7efblod-object@00000000b9cd7efb
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000448bb290osd-ldiskfs-object@00000000448bb290(i:00000000c5dc08ae:188/2305117512)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000009005dc8c
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@0000000091d02677[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000df4be1c4
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000563b300fosd-ldiskfs-object@00000000563b300f(i:0000000008e754c9:81/444013660)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@0000000091d02677
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000005482dc68[0x5, 1, [0x200003ab1:0x2e:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b2074af9mdt-object@000000005482dc68( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000741d0622mdd-object@00000000741d0622(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000008690c980lod-object@000000008690c980
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000816e1f4fosd-ldiskfs-object@00000000816e1f4f(i:0000000049330297:213/1862212438)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000005482dc68
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000c80b7318[0x5, 1, [0x200003ab1:0x1c:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000002f35e52cmdt-object@00000000c80b7318( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000093cdedc4mdd-object@0000000093cdedc4(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000006f0936f9lod-object@000000006f0936f9
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000432060e6osd-ldiskfs-object@00000000432060e6(i:00000000279789cd:195/4070398574)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000c80b7318
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000873ec095[0x5, 1, [0x200003ab1:0x28:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000c1788a11mdt-object@00000000873ec095( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000098a5dbe1mdd-object@0000000098a5dbe1(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000090ea5415lod-object@0000000090ea5415
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000003dbb3bb5osd-ldiskfs-object@000000003dbb3bb5(i:00000000cd5e645b:207/3169949720)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000873ec095
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000007f4a8823[0x5, 1, [0x200003ab1:0x1e:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000006048eaefmdt-object@000000007f4a8823( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000006bcf019fmdd-object@000000006bcf019f(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000005687accflod-object@000000005687accf
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000000dae759eosd-ldiskfs-object@000000000dae759e(i:00000000213b4be5:197/1287571732)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000007f4a8823
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000000ac92cf3[0x5, 1, [0x200003ab1:0x13:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000081630b77mdt-object@000000000ac92cf3( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000003aff604bmdd-object@000000003aff604b(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000003301537lod-object@0000000003301537
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004cb87df9osd-ldiskfs-object@000000004cb87df9(i:00000000201f6feb:186/80712356)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000000ac92cf3
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000521549e9[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000058c3664e
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000006684bd0eosd-ldiskfs-object@000000006684bd0e(i:000000004548d0f7:89/3604252189)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000521549e9
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000007039e03d[0x5, 1, [0x200003ab1:0x24:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000d731344bmdt-object@000000007039e03d( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000960fc2d2mdd-object@00000000960fc2d2(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000003d4c5f8lod-object@0000000003d4c5f8
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000fcbc32a6osd-ldiskfs-object@00000000fcbc32a6(i:0000000083a62090:203/636517153)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000007039e03d
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000cf2a77a9[0x5, 1, [0x200003ab1:0x1a:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000003a016621mdt-object@00000000cf2a77a9( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000008c603076mdd-object@000000008c603076(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000305400aelod-object@00000000305400ae
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000e728103fosd-ldiskfs-object@00000000e728103f(i:00000000749957cf:193/2636114498)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000cf2a77a9
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@0000000078aeedfd[0x5, 1, [0x200003ab1:0x1d:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000236c8898mdt-object@0000000078aeedfd( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000dfe3cc87mdd-object@00000000dfe3cc87(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000923b7eb1lod-object@00000000923b7eb1
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000096a0849fosd-ldiskfs-object@0000000096a0849f(i:00000000f2f9bf94:196/2967340104)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@0000000078aeedfd
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000b5ae67ce[0x5, 1, [0x200003ab1:0x29:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b34c2081mdt-object@00000000b5ae67ce( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000008b14b935mdd-object@000000008b14b935(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a3cac47clod-object@00000000a3cac47c
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000274c96e2osd-ldiskfs-object@00000000274c96e2(i:000000002bd616ae:208/178528692)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000b5ae67ce
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000a407a0f3[0x5, 1, [0x200003ab1:0x1f:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000dc659b1cmdt-object@00000000a407a0f3( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000058069e5emdd-object@0000000058069e5e(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a3ea9df8lod-object@00000000a3ea9df8
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000871f1118osd-ldiskfs-object@00000000871f1118(i:000000005520361f:198/3634217643)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000a407a0f3
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000000bdf9e3c[0x5, 1, [0x200003ab1:0x22:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000008839ac91mdt-object@000000000bdf9e3c( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000703c105emdd-object@00000000703c105e(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000003ca4e422lod-object@000000003ca4e422
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004187dd1fosd-ldiskfs-object@000000004187dd1f(i:000000008f4a9448:201/1608114076)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000000bdf9e3c
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000d6ba78bb[0x5, 1, [0x200003ab1:0x10:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000e7a8d04amdt-object@00000000d6ba78bb( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000008c47fb32mdd-object@000000008c47fb32(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000006ee81a6blod-object@000000006ee81a6b
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000374e2502osd-ldiskfs-object@00000000374e2502(i:000000003a332fa3:183/103855568)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000d6ba78bb
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000003cee58aa[0x5, 1, [0x200003ab1:0x25:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000036637673mdt-object@000000003cee58aa( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000b1421067mdd-object@00000000b1421067(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000004e16f5f4lod-object@000000004e16f5f4
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000c23f13b0osd-ldiskfs-object@00000000c23f13b0(i:0000000004be9835:204/1023749690)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000003cee58aa
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000df18767f[0x5, 1, [0x200003ab1:0x1b:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000005901a312mdt-object@00000000df18767f( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000c26c6a64mdd-object@00000000c26c6a64(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a7c20893lod-object@00000000a7c20893
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000a128b0baosd-ldiskfs-object@00000000a128b0ba(i:000000002970ba7e:194/3215368860)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000df18767f
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000d1ebe709[0x5, 1, [0x200003ab1:0x2c:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000002d012f4emdt-object@00000000d1ebe709( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000c23c5596mdd-object@00000000c23c5596(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000437b61f6lod-object@00000000437b61f6
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000008031839osd-ldiskfs-object@0000000008031839(i:000000008a967bbb:211/1165709955)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000d1ebe709
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000008357c06b[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000836abfae
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000619cca36osd-ldiskfs-object@00000000619cca36(i:00000000613d258a:82/764676917)[lfix]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000008357c06b
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000f6f2a9df[0x5, 1, [0x200003ab1:0x21:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000004fbe7b8mdt-object@00000000f6f2a9df( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000f17831f4mdd-object@00000000f17831f4(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000833173a0lod-object@00000000833173a0
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000ccce9691osd-ldiskfs-object@00000000ccce9691(i:000000009d3e703e:200/1439768907)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000f6f2a9df
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000e96c598f[0x5, 1, [0x200003ab1:0x17:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000045cf1bc5mdt-object@00000000e96c598f( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000935bac9fmdd-object@00000000935bac9f(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000004d3daa63lod-object@000000004d3daa63
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000af115f09osd-ldiskfs-object@00000000af115f09(i:0000000087c91ea4:190/964581435)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000e96c598f
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000003e35d90f[0x5, 1, [0x200003ab1:0x23:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000dd422bcdmdt-object@000000003e35d90f( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000009426c8dbmdd-object@000000009426c8db(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000067dff25lod-object@00000000067dff25
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000001b3fe8c8osd-ldiskfs-object@000000001b3fe8c8(i:00000000d9c53a36:202/3766926486)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000003e35d90f
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000350e080a[0x5, 1, [0x200003ab1:0x11:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000002effd215mdt-object@00000000350e080a( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000091f1c05cmdd-object@0000000091f1c05c(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000008d6acb0alod-object@000000008d6acb0a
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000068036425osd-ldiskfs-object@0000000068036425(i:000000002cdab267:184/3810406434)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000350e080a
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000005640cc66[0x5, 1, [0x200003ab1:0x2a:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000004d7be995mdt-object@000000005640cc66( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000034617ed9mdd-object@0000000034617ed9(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000037f1fcd9lod-object@0000000037f1fcd9
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000069cdcbb4osd-ldiskfs-object@0000000069cdcbb4(i:00000000deabc106:209/1679335188)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000005640cc66
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000009da335f5[0x5, 1, [0x200003ab1:0x18:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000712e0500mdt-object@000000009da335f5( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000c3124101mdd-object@00000000c3124101(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000bb190963lod-object@00000000bb190963
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004aa69970osd-ldiskfs-object@000000004aa69970(i:000000000115f81b:191/2791882283)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000009da335f5
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000005e6081b1[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000b85a9d21
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000b818584bosd-ldiskfs-object@00000000b818584b(i:0000000019b286fb:83/1613481103)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000005e6081b1
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000002b47fd2d[0x5, 1, [0x200003ab1:0x2d:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@000000003c7a277amdt-object@000000002b47fd2d( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000005ab2e986mdd-object@000000005ab2e986(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@0000000096f55657lod-object@0000000096f55657
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000005be04971osd-ldiskfs-object@000000005be04971(i:00000000400d6808:212/774043358)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000002b47fd2d
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000004b875bf8[0x5, 1, [0x200003ab1:0x26:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b40de1b0mdt-object@000000004b875bf8( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@000000009c807feemdd-object@000000009c807fee(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@00000000696f18f0lod-object@00000000696f18f0
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000009359e990osd-ldiskfs-object@000000009359e990(i:000000005bb6861c:205/218859910)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000004b875bf8
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@000000004f7d74c7[0x5, 1, [0x200003ab1:0x14:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000768981efmdt-object@000000004f7d74c7( , writecount=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000a119ff91mdd-object@00000000a119ff91(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....lod@000000008b81ce1blod-object@000000008b81ce1b
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000cf6f621eosd-ldiskfs-object@00000000cf6f621e(i:000000001ded8716:187/1877211335)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@000000004f7d74c7
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) header@00000000aca1be69[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000008f128992
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000009e90e13osd-ldiskfs-object@0000000009e90e13(i:000000002299c6f4:20036/3837903555)[plain]
LustreError: 156698:0:(lu_object.c:984:lu_site_print()) } header@00000000aca1be69

LustreError: 629223:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff8be93c5ed000
LustreError: 629223:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 0 PID: 629223 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f0724a218fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 629223:0:(obd_class.h:479:obd_check_dev()) Device 14 not setup
LustreError: 629223:0:(obd_class.h:479:obd_check_dev()) Skipped 17 previous similar messages
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@00000000f8849224[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000002cc00d1f
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000f5f1b646osd-ldiskfs-object@00000000f5f1b646(i:00000000ff3cd6d6:112/3214040024)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@00000000f8849224
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@000000006576cfe9[0x5, 1, [0x200004281:0x80ee:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000c065cb00mdt-object@000000006576cfe9( , writecount=0)
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000dbfaf8ddmdd-object@00000000dbfaf8dd(open_count=0, valid=0, cltime=0ns, flags=1)
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....lod@00000000b55c2702lod-object@00000000b55c2702
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004f9bd2c7osd-ldiskfs-object@000000004f9bd2c7(i:00000000e12f11a9:486/775406226)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@000000006576cfe9
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@00000000918d3a55[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000001bbaaa5d
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000564c0745osd-ldiskfs-object@00000000564c0745(i:00000000aae86d29:32265/4105829632)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@00000000918d3a55
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@0000000037acaa93[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000076cfd047
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000edd1e2bcosd-ldiskfs-object@00000000edd1e2bc(i:00000000900736fb:32267/2999533703)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@0000000037acaa93
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@000000000a346698[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000021edda14
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000007eae4091osd-ldiskfs-object@000000007eae4091(i:000000002e5de2b7:181/3701438880)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@000000000a346698
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@00000000ea80755e[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
Lustre: lustre-MDT0000: Not available for connect from 10.240.40.110@tcp (stopping)
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000001b831d09
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000003a2324a6osd-ldiskfs-object@000000003a2324a6(i:0000000087c2e4f9:113/666724991)[lfix]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@00000000ea80755e
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) header@0000000023164ad0[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000a8581d7c
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000300e2f6eosd-ldiskfs-object@00000000300e2f6e(i:00000000ea3bce25:147/2769822645)[plain]
LustreError: 629223:0:(lu_object.c:984:lu_site_print()) } header@0000000023164ad0

LustreError: 409613:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff88e4cea4a000
LustreError: 409613:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 409613 Comm: umount Kdump: loaded Tainted: P OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x125/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7faa6eda88fb

Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n mdt.lustre-MDT0000.hsm.actions | awk '/'0x200006999:0x2c5:0x0'.*action='ARCHIVE'/ {print $13}' | cut -f2 -d=
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_val=5 fail_loc=0x164
LustreError: 308794:0:(mdt_hsm_cdt_client.c:366:mdt_hsm_register_hal()) cfs_fail_timeout id 164 sleeping for 5000ms
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param mdt.lustre-MDT0000.hsm.actions
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
Lustre: lustre-MDT0000: Not available for connect from 10.240.22.105@tcp (stopping)
Lustre: Skipped 7 previous similar messages
LustreError: 308794:0:(mdt_hsm_cdt_client.c:366:mdt_hsm_register_hal()) cfs_fail_timeout id 164 awake
LustreError: 409613:0:(obd_class.h:479:obd_check_dev()) Device 18 not setup
LustreError: 409613:0:(obd_class.h:479:obd_check_dev()) Skipped 23 previous similar messages
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@000000004fa3197b[0x5, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000cb66b72f
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000001d60beedosd-zfs-object@000000001d60beed
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@000000004fa3197b
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@0000000029e21436[0x5, 2, [0x200006999:0x2c4:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000024c74479mdt-object@0000000029e21436( , writecount=0)
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000040752abbmdd-object@0000000040752abb(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a1b79dd8lod-object@00000000a1b79dd8
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000092cd31dosd-zfs-object@00000000092cd31d
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@0000000029e21436
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@000000009013d1de[0x4, 1, [0xffffffff:0x1:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000e8cc7845osd-zfs-object@00000000e8cc7845
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@000000009013d1de
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@00000000e2a2bf33[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000001a0108b0
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000002a6b8d96osd-zfs-object@000000002a6b8d96
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@00000000e2a2bf33
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@0000000007124f8e[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000005ddf16a3
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000040907babosd-zfs-object@0000000040907bab
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@0000000007124f8e
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@000000002ec612a4[0x4, 1, [0x200000003:0x4:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000190ec6c2
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000005952030aosd-zfs-object@000000005952030a
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@000000002ec612a4
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@00000000944863ef[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000a7c9cdbc
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000007c7d207aosd-zfs-object@000000007c7d207a
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@00000000944863ef
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) header@00000000b77e8e4c[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000aa3534fd
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000f724cf87osd-zfs-object@00000000f724cf87
LustreError: 409613:0:(lu_object.c:984:lu_site_print()) } header@00000000b77e8e4c

LustreError: 442994:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff9369fa5cb000
LustreError: 442994:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 0 PID: 442994 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x125/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7fb7a36f98fb

Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n mdt.lustre-MDT0000.hsm.actions | awk '/'0x200007939:0x14c:0x0'.*action='ARCHIVE'/ {print $13}' | cut -f2 -d=
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_val=5 fail_loc=0x164
LustreError: 339977:0:(mdt_hsm_cdt_client.c:366:mdt_hsm_register_hal()) cfs_fail_timeout id 164 sleeping for 5000ms
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param mdt.lustre-MDT0000.hsm.actions
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
Lustre: lustre-MDT0000: Not available for connect from 10.240.23.206@tcp (stopping)
Lustre: Skipped 8 previous similar messages
LustreError: lustre-MDT0000-osp-MDT0002: operation mds_statfs to node 0@lo failed: rc = -107
Lustre: lustre-MDT0000-osp-MDT0002: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: Skipped 1 previous similar message
Lustre: Skipped 5 previous similar messages
LustreError: 339977:0:(mdt_hsm_cdt_client.c:366:mdt_hsm_register_hal()) cfs_fail_timeout id 164 awake
LustreError: 442994:0:(obd_class.h:479:obd_check_dev()) Device 18 not setup
LustreError: 442994:0:(obd_class.h:479:obd_check_dev()) Skipped 23 previous similar messages
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@00000000f4c8f34c[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000003aa10146
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000b305b8c6osd-ldiskfs-object@00000000b305b8c6(i:000000007579bf6a:113/1390987671)[lfix]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@00000000f4c8f34c
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@00000000a45bd055[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000f9b15341
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000003e0dc1c4osd-ldiskfs-object@000000003e0dc1c4(i:000000002cdae903:31713/1906262206)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@00000000a45bd055
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@00000000afc8abf1[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000025f93732
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000001392d248osd-ldiskfs-object@000000001392d248(i:000000008bbcb4af:31715/3548344995)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@00000000afc8abf1
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@000000002a2a2dd6[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000056f18f19
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000f685ff2cosd-ldiskfs-object@00000000f685ff2c(i:000000007974ed10:147/4174049612)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@000000002a2a2dd6
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@000000006fa3f5d4[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000220a96a2
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000006aff2a99osd-ldiskfs-object@000000006aff2a99(i:00000000206bfb9a:112/216793792)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@000000006fa3f5d4
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@000000001af01537[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000004e120748
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000bb13c290osd-ldiskfs-object@00000000bb13c290(i:00000000e1b553f5:181/3639469567)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@000000001af01537
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) header@000000002ec825be[0x5, 2, [0x200007939:0x14a:0x0] hash exist]{
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....mdt@000000008d3d6c11mdt-object@000000002ec825be( , writecount=0)
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....mdd@000000009d642e35mdd-object@000000009d642e35(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....lod@000000003f010be1lod-object@000000003f010be1
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000081ca584cosd-ldiskfs-object@0000000081ca584c(i:000000000e85452a:778/906609427)[plain]
LustreError: 442994:0:(lu_object.c:984:lu_site_print()) } header@000000002ec825be

LustreError: 635488:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff92e5502e6000
LustreError: 635488:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 0 PID: 635488 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7fd838ea78fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 635488:0:(obd_class.h:479:obd_check_dev()) Device 14 not setup
LustreError: 635488:0:(obd_class.h:479:obd_check_dev()) Skipped 17 previous similar messages
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@00000000884b0c0e[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000071c448f9
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000003c082780osd-ldiskfs-object@000000003c082780(i:00000000ff7a847b:112/1943606573)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@00000000884b0c0e
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@00000000fec6f45e[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000009f27315d
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000005583a3b3osd-ldiskfs-object@000000005583a3b3(i:00000000fce18d1d:32265/1130482448)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@00000000fec6f45e
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@000000001c2f5588[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000009804bdd3
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000f01be25fosd-ldiskfs-object@00000000f01be25f(i:00000000d3a09d1f:181/2828124433)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@000000001c2f5588
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@000000001a53d0b3[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000000f6f0c0f
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000ce137603osd-ldiskfs-object@00000000ce137603(i:0000000072a155b8:32267/70280480)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@000000001a53d0b3
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@00000000b487c021[0x5, 1, [0x200004281:0x80ee:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....mdt@000000009001418bmdt-object@00000000b487c021( , writecount=0)
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....mdd@000000007d093667mdd-object@000000007d093667(open_count=0, valid=0, cltime=0ns, flags=1)
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....lod@00000000e485b840lod-object@00000000e485b840
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000080d99f6eosd-ldiskfs-object@0000000080d99f6e(i:00000000c450f55d:486/300240793)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@00000000b487c021
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@00000000290d38ae[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000095230ed7
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000f5c51327osd-ldiskfs-object@00000000f5c51327(i:0000000054759aee:113/2071094071)[lfix]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@00000000290d38ae
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) header@000000000cf339f4[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000039b38799
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000d0cf7888osd-ldiskfs-object@00000000d0cf7888(i:000000005a1ba4b1:147/1846639465)[plain]
LustreError: 635488:0:(lu_object.c:984:lu_site_print()) } header@000000000cf339f4

LustreError: 115402:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff9fa7f262f000
LustreError: 115402:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 0 PID: 115402 Comm: umount Kdump: loaded Tainted: P OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x125/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f2e76bba8fb

Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0x1709
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param fail_loc=0
Lustre: DEBUG MARKER: /usr/sbin/lctl lfsck_start -M lustre-MDT0000 -t namespace -A -r
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d -f /mnt/lustre-mds1
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@0000000096d84831[0x5, 1, [0x200003ab1:0x1c:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000d5143084mdt-object@0000000096d84831( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000a1a167b9mdd-object@00000000a1a167b9(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000e8320e9flod-object@00000000e8320e9f
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000d1556493osd-zfs-object@00000000d1556493
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@0000000096d84831
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000c962b710[0x5, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000010da0945
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000d7bb3d29osd-zfs-object@00000000d7bb3d29
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000c962b710
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000d4a0329b[0x5, 1, [0x200003ab1:0x17:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b6f8ac12mdt-object@00000000d4a0329b( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000019d7d0ddmdd-object@0000000019d7d0dd(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000069cf645blod-object@0000000069cf645b
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000089879e55osd-zfs-object@0000000089879e55
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000d4a0329b
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000f6d5da24[0x5, 1, [0x200003ab1:0x10:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000009cfc9790mdt-object@00000000f6d5da24( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000082cc3e70mdd-object@0000000082cc3e70(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a6706c9dlod-object@00000000a6706c9d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000919dd863osd-zfs-object@00000000919dd863
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000f6d5da24
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000002334438b[0x5, 1, [0x200003ab1:0x19:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000009b35ca44mdt-object@000000002334438b( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000005239984cmdd-object@000000005239984c(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000421f54e3lod-object@00000000421f54e3
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000008c72dc39osd-zfs-object@000000008c72dc39
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000002334438b
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@0000000053bec799[0x5, 1, [0x200003ab1:0x14:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000f6f3a7c6mdt-object@0000000053bec799( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000037bbfe47mdd-object@0000000037bbfe47(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000081b88d68lod-object@0000000081b88d68
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000447287afosd-zfs-object@00000000447287af
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@0000000053bec799
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000e76868a7[0x5, 1, [0x200003ab1:0x22:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000001cdbc056mdt-object@00000000e76868a7( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000093d1b853mdd-object@0000000093d1b853(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000b8020a11lod-object@00000000b8020a11
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000000b43c6f0osd-zfs-object@000000000b43c6f0
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000e76868a7
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@0000000048402440[0x5, 1, [0x200003ab1:0x16:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000688335c6mdt-object@0000000048402440( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000e62e0bfemdd-object@00000000e62e0bfe(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@000000001bfcbb5flod-object@000000001bfcbb5f
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000000301fcb3osd-zfs-object@000000000301fcb3
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@0000000048402440
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000bd3cda51[0x5, 1, [0x200003ab1:0x24:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000964f9ae0mdt-object@00000000bd3cda51( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000a3064890mdd-object@00000000a3064890(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000086d752f7lod-object@0000000086d752f7
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000006ea56a6osd-zfs-object@0000000006ea56a6
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000bd3cda51
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000000f39a82a[0x4, 1, [0xffffffff:0x1:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000006b2a6032osd-zfs-object@000000006b2a6032
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000000f39a82a
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@0000000022ab3e43[0x5, 1, [0x200003ab1:0x1f:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000007b0c055mdt-object@0000000022ab3e43( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000594512dbmdd-object@00000000594512db(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a59be25clod-object@00000000a59be25c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000c765b0a1osd-zfs-object@00000000c765b0a1
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@0000000022ab3e43
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000b52b72db[0x5, 1, [0x200003ab1:0x18:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000805a2ac9mdt-object@00000000b52b72db( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000005b64b067mdd-object@000000005b64b067(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000037a4fc0lod-object@00000000037a4fc0
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000d2846cdeosd-zfs-object@00000000d2846cde
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000b52b72db
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000005269bb7d[0x5, 1, [0x200003ab1:0x13:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000852bdb4dmdt-object@000000005269bb7d( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000003b099a9mdd-object@0000000003b099a9(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000515f6c69lod-object@00000000515f6c69
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000040873361osd-zfs-object@0000000040873361
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000005269bb7d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000fcb455f3[0x5, 1, [0x200003ab1:0x21:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000001028f94dmdt-object@00000000fcb455f3( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000005929e409mdd-object@000000005929e409(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@000000000cc5f040lod-object@000000000cc5f040
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000008bd85d71osd-zfs-object@000000008bd85d71
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000fcb455f3
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000003648152c[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000024a6c1ee
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000007421d729osd-zfs-object@000000007421d729
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000003648152c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000002a5aa2e3[0x5, 1, [0x200003ab1:0x15:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000491babe6mdt-object@000000002a5aa2e3( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000c8e7afb6mdd-object@00000000c8e7afb6(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000f9118024lod-object@00000000f9118024
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000261e5369osd-zfs-object@00000000261e5369
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000002a5aa2e3
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000c8b501cc[0x5, 1, [0x200003ab1:0x23:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000da1e5bb7mdt-object@00000000c8b501cc( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000cbf4f858mdd-object@00000000cbf4f858(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000029861498lod-object@0000000029861498
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000004934c3a2osd-zfs-object@000000004934c3a2
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000c8b501cc
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000e91d8d64[0x5, 1, [0x200003ab1:0x1e:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000002724f926mdt-object@00000000e91d8d64( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000003128f4d6mdd-object@000000003128f4d6(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000f55605cblod-object@00000000f55605cb
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000c75bd359osd-zfs-object@00000000c75bd359
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000e91d8d64
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000006c88051d[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000b5513688
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000046a711d2osd-zfs-object@0000000046a711d2
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000006c88051d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000006a8ce7af[0x5, 1, [0x200003ab1:0x12:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000bf6e8efemdt-object@000000006a8ce7af( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000007bcbfedfmdd-object@000000007bcbfedf(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000eaf986a2lod-object@00000000eaf986a2
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000000d08b04dosd-zfs-object@000000000d08b04d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000006a8ce7af
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000f02a6f29[0x5, 1, [0x200003ab1:0x20:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000ed0d6575mdt-object@00000000f02a6f29( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000e8a6758emdd-object@00000000e8a6758e(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000e6a5c44dlod-object@00000000e6a5c44d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000006f885727osd-zfs-object@000000006f885727
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000f02a6f29
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000698115fa[0x5, 1, [0x200003ab1:0x1b:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000025bee24bmdt-object@00000000698115fa( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000006d2c0fa1mdd-object@000000006d2c0fa1(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000e04f3e2clod-object@00000000e04f3e2c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000003d5d463aosd-zfs-object@000000003d5d463a
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000698115fa
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000003e9f5802[0x4, 1, [0x200000003:0x4:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000add5f9ff
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000deb1523cosd-zfs-object@00000000deb1523c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000003e9f5802
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@000000004563a750[0x5, 1, [0x200003ab1:0xf:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000dd43e2d6mdt-object@000000004563a750( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000009e3ceb62mdd-object@000000009e3ceb62(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000dac1383clod-object@00000000dac1383c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000e15c27d1osd-zfs-object@00000000e15c27d1
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@000000004563a750
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000034bb039[0x5, 1, [0x200003ab1:0x1d:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@000000008e2c83b1mdt-object@00000000034bb039( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@000000001133e349mdd-object@000000001133e349(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000034b700b9lod-object@0000000034b700b9
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000b323bb02osd-zfs-object@00000000b323bb02
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000034bb039
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000448ca8a9[0x5, 1, [0x200003ab1:0x11:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000043b268cdmdt-object@00000000448ca8a9( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000082b17d2bmdd-object@0000000082b17d2b(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@00000000ac5da70elod-object@00000000ac5da70e
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000002a02ea4dosd-zfs-object@000000002a02ea4d
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000448ca8a9
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@0000000008166047[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000b0a977ae
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000005f61fd2cosd-zfs-object@000000005f61fd2c
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@0000000008166047
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000d52ce4d5[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000f7e61bee
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000019212589osd-zfs-object@0000000019212589
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000d52ce4d5
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) header@00000000ec49b245[0x5, 1, [0x200003ab1:0x1a:0x0] hash exist]{
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000febdb5c3mdt-object@00000000ec49b245( , writecount=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000091737ca6mdd-object@0000000091737ca6(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....lod@0000000059cf8bcflod-object@0000000059cf8bcf
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000009be9b5b0osd-zfs-object@000000009be9b5b0
LustreError: 115402:0:(lu_object.c:984:lu_site_print()) } header@00000000ec49b245

LustreError: 625644:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff9181edf20000
LustreError: 625644:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 625644 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f918b0d88fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 625644:0:(obd_class.h:479:obd_check_dev()) Device 14 not setup
LustreError: 625644:0:(obd_class.h:479:obd_check_dev()) Skipped 17 previous similar messages
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@00000000ff700306[0x5, 1, [0x200004281:0x80ee:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....mdt@000000008fffebdbmdt-object@00000000ff700306( , writecount=0)
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000005789d61mdd-object@0000000005789d61(open_count=0, valid=0, cltime=0ns, flags=1)
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....lod@00000000aeb0e6dclod-object@00000000aeb0e6dc
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000c99813fbosd-ldiskfs-object@00000000c99813fb(i:000000003bf5ee89:486/1743478810)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@00000000ff700306
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@00000000ef63e1f7[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000628f1f2f
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000dbbf569fosd-ldiskfs-object@00000000dbbf569f(i:000000008a6f11bb:181/3011035148)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@00000000ef63e1f7
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@00000000fdc25e8c[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000006c692c01
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000abc05ecaosd-ldiskfs-object@00000000abc05eca(i:00000000f950b5e2:112/1936010309)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@00000000fdc25e8c
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@0000000091b73369[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000003f7ab9de
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000009508479fosd-ldiskfs-object@000000009508479f(i:000000007dda35bd:32267/2946757870)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@0000000091b73369
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@00000000814a9c82[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000482b3d54
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000009ed35607osd-ldiskfs-object@000000009ed35607(i:00000000353523f9:147/2499219793)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@00000000814a9c82
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@000000001003b8b9[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000006876f268
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000aefff880osd-ldiskfs-object@00000000aefff880(i:0000000066ca0e6b:113/193423531)[lfix]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@000000001003b8b9
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) header@00000000b7a39ee8[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000cb1a564f
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000a5fe8b1aosd-ldiskfs-object@00000000a5fe8b1a(i:00000000c74bf9d2:32265/4094053224)[plain]
LustreError: 625644:0:(lu_object.c:984:lu_site_print()) } header@00000000b7a39ee8

LustreError: 597515:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff9da23da05000
LustreError: 597515:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 597515 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f832b28c8fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 597515:0:(obd_class.h:479:obd_check_dev()) Device 14 not setup
LustreError: 597515:0:(obd_class.h:479:obd_check_dev()) Skipped 17 previous similar messages
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@0000000038d785ef[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000002cb2dafc
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000022102310osd-ldiskfs-object@0000000022102310(i:00000000fb24f2e6:32267/1045942643)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@0000000038d785ef
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@000000003da193ef[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000027983fa1
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000001fbf3bbdosd-ldiskfs-object@000000001fbf3bbd(i:000000008cb58d13:147/8379686)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@000000003da193ef
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@000000007a20768e[0x5, 1, [0x200004281:0x80ee:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000b5cf6e4cmdt-object@000000007a20768e( , writecount=0)
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000ddedcf5cmdd-object@00000000ddedcf5c(open_count=0, valid=0, cltime=0ns, flags=1)
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....lod@0000000031a95686lod-object@0000000031a95686
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000dbde8e37osd-ldiskfs-object@00000000dbde8e37(i:00000000175ac2d9:486/561500356)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@000000007a20768e
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@00000000c2f8c74f[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000000e5eb01f
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004990b22dosd-ldiskfs-object@000000004990b22d(i:000000001bfef746:32265/1505062847)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@00000000c2f8c74f
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@000000001c96bf4a[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000e28b5638
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000007e65c506osd-ldiskfs-object@000000007e65c506(i:00000000ad9117f0:113/2754187631)[lfix]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@000000001c96bf4a
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@0000000027cb7e9c[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000006191df8e
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@000000004e01dfbfosd-ldiskfs-object@000000004e01dfbf(i:00000000203adf5b:181/1566393250)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@0000000027cb7e9c
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) header@000000003bae3d7b[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000007483fd78
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000bf266461osd-ldiskfs-object@00000000bf266461(i:000000000a17adbc:112/3451739009)[plain]
LustreError: 597515:0:(lu_object.c:984:lu_site_print()) } header@000000003bae3d7b

LustreError: 575444:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff96906eabe000
LustreError: 575444:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 575444 Comm: umount Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f9f5de018fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 575444:0:(obd_class.h:479:obd_check_dev()) Device 14 not setup
LustreError: 575444:0:(obd_class.h:479:obd_check_dev()) Skipped 17 previous similar messages
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@000000003eb05ad0[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000cd817128
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000fa2e31b0osd-ldiskfs-object@00000000fa2e31b0(i:000000009885bf1b:32267/1694457204)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@000000003eb05ad0
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@00000000bed5f25c[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000004157c1b
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000046afbdc1osd-ldiskfs-object@0000000046afbdc1(i:00000000d988e8f0:147/696169257)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@00000000bed5f25c
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@000000009e374880[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@0000000056f40063
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000004b7f76cosd-ldiskfs-object@0000000004b7f76c(i:00000000119d8a27:113/3920903360)[lfix]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@000000009e374880
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@00000000016e9757[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000c80b7ec0
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@0000000078ef1de3osd-ldiskfs-object@0000000078ef1de3(i:000000002a86cb48:32265/175254257)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@00000000016e9757
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@00000000fa6eb389[0x5, 1, [0x200004281:0x80ee:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....mdt@000000001f38da37mdt-object@00000000fa6eb389( , writecount=0)
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000a91ef8edmdd-object@00000000a91ef8ed(open_count=0, valid=0, cltime=0ns, flags=1)
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....lod@0000000032797a83lod-object@0000000032797a83
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000510d046aosd-ldiskfs-object@00000000510d046a(i:0000000064f517c4:486/344557037)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@00000000fa6eb389
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@000000000a1b361e[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000c360508b
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000447679b3osd-ldiskfs-object@00000000447679b3(i:00000000c61448e1:181/1656474581)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@000000000a1b361e
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) header@00000000b1a94ba8[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000000c00cc7b
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) ....osd-ldiskfs@00000000e8be13d6osd-ldiskfs-object@00000000e8be13d6(i:000000003e7f3262:112/106429610)[plain]
LustreError: 575444:0:(lu_object.c:984:lu_site_print()) } header@00000000b1a94ba8

LustreError: 377764:0:(lod_dev.c:2162:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff8a43fc4b2000
LustreError: 377764:0:(lod_dev.c:2162:lod_device_free()) LBUG
CPU: 1 PID: 377764 Comm: umount Kdump: loaded Tainted: P OE -------- - - 4.18.0-553.53.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x257/0x260 [lod]
class_free_dev+0x5d5/0x710 [obdclass]
class_detach+0x288/0x2d0 [obdclass]
class_process_config+0x16cc/0x20f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x478/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x313/0x650 [mdd]
mdt_stack_fini+0x2b0/0xad0 [mdt]
mdt_device_fini+0xb43/0xef0 [mdt]
obd_precleanup+0xdc/0x280 [obdclass]
? class_disconnect_exports+0x187/0x2f0 [obdclass]
class_cleanup+0x322/0x7e0 [obdclass]
class_process_config+0x3bb/0x20f0 [obdclass]
class_manual_cleanup+0x45b/0x780 [obdclass]
server_put_super+0xd65/0x1440 [ptlrpc]
? fsnotify_sb_delete+0x138/0x1c0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x195/0x1a0
entry_SYSCALL_64_after_hwframe+0x66/0xcb
RIP: 0033:0x7f2336d2c8fb

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d -f /mnt/lustre-mds1
Lustre: lustre-MDT0000: Not available for connect from 10.240.40.10@tcp (stopping)
Lustre: Skipped 3 previous similar messages
Lustre: lustre-MDT0000-osp-MDT0002: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0000-osp-MDT0002: operation mds_statfs to node 0@lo failed: rc = -107
Lustre: Skipped 5 previous similar messages
Lustre: lustre-MDT0000: Not available for connect from 0@lo (stopping)
LustreError: Skipped 1 previous similar message
Lustre: Skipped 8 previous similar messages
LustreError: 377764:0:(obd_class.h:479:obd_check_dev()) Device 33 not setup
LustreError: 377764:0:(obd_class.h:479:obd_check_dev()) Skipped 23 previous similar messages
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@000000000c15d950[0x5, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000003ef7e8a6
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000045e4ab06osd-zfs-object@0000000045e4ab06
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@000000000c15d950
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000156c77a9[0x5, 1, [0x200000407:0xe346:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@000000005b33b6aamdt-object@00000000156c77a9( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000b21acee1mdd-object@00000000b21acee1(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@00000000e8c6b369lod-object@00000000e8c6b369
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000052ba05d5osd-zfs-object@0000000052ba05d5
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000156c77a9
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@0000000002a60a48[0x5, 1, [0x200000407:0xe349:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000ab004300mdt-object@0000000002a60a48( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@0000000084e7a636mdd-object@0000000084e7a636(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@00000000581ee268lod-object@00000000581ee268
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000aaab8d11osd-zfs-object@00000000aaab8d11
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@0000000002a60a48
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000fda58411[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000b37e48a0
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000e76a0ffaosd-zfs-object@00000000e76a0ffa
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000fda58411
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@000000008bd14a93[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000bc144951
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@0000000091a3512cosd-zfs-object@0000000091a3512c
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@000000008bd14a93
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000acd72bdf[0x5, 1, [0x200000407:0xe344:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@00000000326a067cmdt-object@00000000acd72bdf( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@000000001803f0c3mdd-object@000000001803f0c3(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@00000000f6223a01lod-object@00000000f6223a01
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000479f13b5osd-zfs-object@00000000479f13b5
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000acd72bdf
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000b0ef5b1d[0x5, 1, [0x200000407:0xe34a:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@0000000059955b54mdt-object@00000000b0ef5b1d( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@000000001b02d7b8mdd-object@000000001b02d7b8(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@00000000a9a2e04flod-object@00000000a9a2e04f
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000dc871759osd-zfs-object@00000000dc871759
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000b0ef5b1d
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000e6af24cb[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000009c31b4b3
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000008a91244eosd-zfs-object@000000008a91244e
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000e6af24cb
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000ece96cfd[0x5, 1, [0x200000407:0xe348:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@000000003249cce4mdt-object@00000000ece96cfd( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000b6f40e22mdd-object@00000000b6f40e22(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@00000000bc3ab74flod-object@00000000bc3ab74f
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000a41670e3osd-zfs-object@00000000a41670e3
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000ece96cfd
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@000000004bf860cf[0x4, 1, [0xffffffff:0x1:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000008db738e3osd-zfs-object@000000008db738e3
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@000000004bf860cf
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000c8e7c280[0x4, 1, [0x200000003:0x4:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@000000009a004240
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@000000001d1bd9e1osd-zfs-object@000000001d1bd9e1
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000c8e7c280
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000cfd1202b[0x5, 1, [0x200000407:0xe345:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdt@000000003ec1ea5fmdt-object@00000000cfd1202b( , writecount=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....mdd@00000000faf51278mdd-object@00000000faf51278(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....lod@0000000011273a95lod-object@0000000011273a95
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000625ad35eosd-zfs-object@00000000625ad35e
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000cfd1202b
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) header@00000000098c0a3e[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....local_storage@00000000301f9ab7
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) ....osd-zfs@00000000a0d0223eosd-zfs-object@00000000a0d0223e
LustreError: 377764:0:(lu_object.c:984:lu_site_print()) } header@00000000098c0a3e

LustreError: 781009:0:(lod_dev.c:2171:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff969caff39000
LustreError: 781009:0:(lod_dev.c:2171:lod_device_free()) LBUG
CPU: 1 PID: 781009 Comm: umount Kdump: loaded Tainted: G OE --------- - - 4.18.0-513.18.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x252/0x260 [lod]
class_free_dev+0x2ef/0x600 [obdclass]
class_detach+0x285/0x2d0 [obdclass]
class_process_config+0x1761/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x471/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x311/0x6f0 [mdd]
mdt_stack_fini+0x2ad/0x6c0 [mdt]
mdt_device_fini+0xa73/0xf60 [mdt]
obd_precleanup+0x1e5/0x220 [obdclass]
class_cleanup+0x31e/0x900 [obdclass]
class_process_config+0x3ad/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x456/0x780 [obdclass]
server_put_super+0x7b6/0x1310 [ptlrpc]
? fsnotify_unmount_inodes+0x11c/0x1b0
? evict_inodes+0x160/0x1b0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x19c/0x1b0
entry_SYSCALL_64_after_hwframe+0x61/0xc6
RIP: 0033:0x7fc01c304e9b

Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osc.*MDT*.sync_*
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osp.*.destroys_in_flight
Lustre: DEBUG MARKER: lctl set_param fail_val=0 fail_loc=0
Lustre: DEBUG MARKER: lctl set_param -n os[cd]*.*MDT*.force_sync=1
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param lustre.quota.ost=none
Autotest: Test running for 240 minutes (lustre-reviews_review-dne-part-4_103807.13)
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param lustre.quota.ost=ugp
LustreError: 780814:0:(qsd_reint.c:635:qqi_reint_delayed()) lustre-MDT0002: Delaying reintegration for qtype:0 until pending updates are flushed.
LustreError: 780814:0:(qsd_reint.c:635:qqi_reint_delayed()) Skipped 11 previous similar messages
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@00000000f48a393e[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000a443ee75
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000c16239b7osd-ldiskfs-object@00000000c16239b7(i:00000000e3de15e1:113/3060834839)[lfix]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@00000000f48a393e
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@00000000079bdb21[0x4, 1, [0x200000006:0x2020000:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....mdt@00000000d62a920cmdt-object@00000000079bdb21( , writecount=0)
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....mdd@000000000e00bf89mdd-object@000000000e00bf89(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....lod@00000000105e67e8lod-object@00000000105e67e8
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000f7a205e1osd-ldiskfs-object@00000000f7a205e1(i:000000007a9b1f27:195/2559859268)[lfix]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@00000000079bdb21
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@000000006d63fba8[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000055a1236c
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000008167fa4dosd-ldiskfs-object@000000008167fa4d(i:000000008eb25176:31713/383567135)[plain]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@000000006d63fba8
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@00000000eea19f68[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000007f87a86e
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000043ad85c6osd-ldiskfs-object@0000000043ad85c6(i:000000002916fc2e:147/1128587623)[plain]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@00000000eea19f68
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@000000005b57e5f9[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000836cc16a
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000090bc32c7osd-ldiskfs-object@0000000090bc32c7(i:00000000f1d8bc0b:112/388356458)[plain]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@000000005b57e5f9
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@00000000ceb8b98d[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000004d949fe9
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000208e54d1osd-ldiskfs-object@00000000208e54d1(i:0000000058e26d7c:31715/2351806035)[plain]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@00000000ceb8b98d
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) header@0000000094c0ea42[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000c4a75dbc
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000019c81275osd-ldiskfs-object@0000000019c81275(i:00000000dc31436e:181/2418830618)[plain]
LustreError: 781009:0:(lu_object.c:998:lu_site_print()) } header@0000000094c0ea42

LustreError: 496111:0:(lod_dev.c:2171:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff8a47f9909000
LustreError: 496111:0:(lod_dev.c:2171:lod_device_free()) LBUG
CPU: 0 PID: 496111 Comm: umount Kdump: loaded Tainted: G OE --------- - - 4.18.0-513.18.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x252/0x260 [lod]
class_free_dev+0x2ef/0x600 [obdclass]
class_detach+0x285/0x2d0 [obdclass]
class_process_config+0x1761/0x21f0 [obdclass]
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x471/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x311/0x6f0 [mdd]
mdt_stack_fini+0x2ad/0x6c0 [mdt]
mdt_device_fini+0xa73/0xf60 [mdt]
? lu_context_init+0xa5/0x1b0 [obdclass]
obd_precleanup+0x1e5/0x220 [obdclass]
class_cleanup+0x31e/0x900 [obdclass]
class_process_config+0x3ad/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
class_manual_cleanup+0x456/0x780 [obdclass]
server_put_super+0x7b6/0x1310 [ptlrpc]
? fsnotify_unmount_inodes+0x11c/0x1b0
? evict_inodes+0x160/0x1b0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x19c/0x1b0
entry_SYSCALL_64_after_hwframe+0x61/0xc6
RIP: 0033:0x7fc2ee8aee9b

Lustre: DEBUG MARKER: which tunefs.lustre
Lustre: DEBUG MARKER: find /usr/lib64/lustre/tests -maxdepth 1 -name 'disk*-ldiskfs.tar.bz2'
Lustre: DEBUG MARKER: /usr/sbin/lctl list_nids
Lustre: DEBUG MARKER: mkdir -p /tmp/t32/mnt/mdt /tmp/t32/mnt/mdt1 /tmp/t32/mnt/ost /tmp/t32/mnt/ost1
Lustre: DEBUG MARKER: tar xjvf /usr/lib64/lustre/tests/disk2_10-ldiskfs.tar.bz2 -S -C /tmp/t32
Lustre: DEBUG MARKER: cat /tmp/t32/commit
Lustre: DEBUG MARKER: cat /tmp/t32/kernel
Lustre: DEBUG MARKER: cat /tmp/t32/arch
Lustre: DEBUG MARKER: cat /tmp/t32/bspace
Lustre: DEBUG MARKER: cat /tmp/t32/ispace
Lustre: DEBUG MARKER: test -f /tmp/t32/blimit
Lustre: DEBUG MARKER: cat /tmp/t32/blimit
Lustre: DEBUG MARKER: test -f /tmp/t32/ilimit
Lustre: DEBUG MARKER: cat /tmp/t32/ilimit
Lustre: DEBUG MARKER: test -f /tmp/t32/mdt2
Lustre: DEBUG MARKER: test -f /tmp/t32/ost2
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug=-1
Lustre: DEBUG MARKER: tunefs.lustre --dryrun /tmp/t32/mdt
Autotest: Test running for 130 minutes (lustre-reviews_review-dne-part-3_103708.12)
Lustre: DEBUG MARKER: tunefs.lustre --dryrun /tmp/t32/mdt2
Lustre: DEBUG MARKER: tunefs.lustre --quota /tmp/t32/mdt
loop0: detected capacity change from 0 to 204800000
Lustre: DEBUG MARKER: tune2fs -O project /tmp/t32/mdt
Lustre: DEBUG MARKER: tunefs.lustre --quota /tmp/t32/mdt2
loop0: detected capacity change from 0 to 204800000
Lustre: DEBUG MARKER: tune2fs -O project /tmp/t32/mdt2
Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u
Lustre: DEBUG MARKER: /usr/sbin/lctl mark onyx-77vm11.onyx.whamcloud.com: executing \/usr\/sbin\/lctl device_list
Lustre: DEBUG MARKER: /usr/sbin/lctl mark onyx-77vm11.onyx.whamcloud.com: executing \/usr\/sbin\/lctl device_list
Lustre: DEBUG MARKER: onyx-77vm11.onyx.whamcloud.com: executing /usr/sbin/lctl device_list
Lustre: DEBUG MARKER: onyx-77vm11.onyx.whamcloud.com: executing /usr/sbin/lctl device_list
Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u
Lustre: DEBUG MARKER: /usr/sbin/lctl mark onyx-77vm11.onyx.whamcloud.com: executing losetup -a
Lustre: DEBUG MARKER: /usr/sbin/lctl mark onyx-77vm11.onyx.whamcloud.com: executing losetup -a
Lustre: DEBUG MARKER: onyx-77vm11.onyx.whamcloud.com: executing losetup -a
Lustre: DEBUG MARKER: onyx-77vm11.onyx.whamcloud.com: executing losetup -a
Lustre: DEBUG MARKER: mount -t lustre -o loop,writeconf /tmp/t32/mdt /tmp/t32/mnt/mdt
loop0: detected capacity change from 0 to 204800000
LDISKFS-fs (loop0): mounted filesystem with ordered data mode. Opts: user_xattr,errors=remount-ro,no_mbcache,nodelalloc
Lustre: t32fs-MDT0000-osd: enabled 'large_dir' feature on device /dev/loop0
Lustre: 491013:0:(scrub.c:191:scrub_file_load()) t32fs-MDT0000: reset scrub OI count for format change (LU-16655)
Lustre: MGS: Logs for fs t32fs were removed by user request. All servers must be restarted in order to regenerate the logs: rc = 0
Lustre: Setting parameter t32fs-MDT0000-mdtlov.lov.stripesize in log t32fs-MDT0000
Lustre: t32fs-MDT0000: Imperative Recovery not enabled, recovery window 60-180
Lustre: DEBUG MARKER: mount -t lustre -o mgsnode=10.240.26.126@tcp,loop,writeconf /tmp/t32/mdt2 /tmp/t32/mnt/mdt1
loop1: detected capacity change from 0 to 204800000
LDISKFS-fs (loop1): mounted filesystem with ordered data mode. Opts: user_xattr,errors=remount-ro,no_mbcache,nodelalloc
Lustre: MGS: Regenerating t32fs-MDT0001 log by user request: rc = 0
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n mdt.t32fs*.enable_remote_dir=1
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n mdt.t32fs-MDT0000.uuid
Lustre: DEBUG MARKER: tunefs.lustre --dryrun /tmp/t32/ost
Lustre: DEBUG MARKER: tunefs.lustre --dryrun /tmp/t32/ost2
Lustre: DEBUG MARKER: tunefs.lustre --quota /tmp/t32/ost
loop2: detected capacity change from 0 to 204800000
Lustre: DEBUG MARKER: tune2fs -O project /tmp/t32/ost
Lustre: DEBUG MARKER: tunefs.lustre --quota /tmp/t32/ost2
loop2: detected capacity change from 0 to 204800000
Lustre: DEBUG MARKER: tune2fs -O project /tmp/t32/ost2
Lustre: DEBUG MARKER: mount -t lustre -onomgs -oloop,mgsnode=10.240.26.126@tcp,writeconf /tmp/t32/ost /tmp/t32/mnt/ost
loop2: detected capacity change from 0 to 204800000
LDISKFS-fs (loop2): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc
Lustre: t32fs-OST0000-osd: enabled 'large_dir' feature on device /dev/loop2
Lustre: Skipped 1 previous similar message
Lustre: 492118:0:(scrub.c:191:scrub_file_load()) t32fs-OST0000: reset scrub OI count for format change (LU-16655)
Lustre: 492118:0:(scrub.c:191:scrub_file_load()) Skipped 1 previous similar message
Lustre: MGS: Regenerating t32fs-OST0000 log by user request: rc = 0
Lustre: t32fs-OST0000: Imperative Recovery not enabled, recovery window 60-180
Lustre: Skipped 1 previous similar message
Lustre: DEBUG MARKER: mount -t lustre -onomgs -oloop,mgsnode=10.240.26.126@tcp,writeconf /tmp/t32/ost2 /tmp/t32/mnt/ost1
loop3: detected capacity change from 0 to 204800000
LDISKFS-fs (loop3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n obdfilter.t32fs-OST0000.uuid
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n obdfilter.t32fs-OST0001.uuid
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-OST0000.failover.node=10.240.26.126@tcp
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0000.failover.node=10.240.26.126@tcp
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-OST0000.osc.max_dirty_mb=15
Lustre: Setting parameter t32fs-OST0000-osc.osc.max_dirty_mb in log t32fs-client
Lustre: Skipped 9 previous similar messages
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0000.mdc.max_rpcs_in_flight=9
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0000.lov.stripesize=4M
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0000.mdd.atime_diff=70
Lustre: DEBUG MARKER: /usr/sbin/lctl pool_new t32fs.interop
Lustre: t32fs-OST0000: deleting orphan objects from 0x280000400:68 to 0x280000400:129
Lustre: t32fs-OST0001: deleting orphan objects from 0x2c0000400:67 to 0x2c0000400:129
Lustre: t32fs-OST0000: deleting orphan objects from 0x0:11 to 0x0:97
Lustre: t32fs-OST0001: deleting orphan objects from 0x0:10 to 0x0:97
Lustre: t32fs-OST0000-osc-MDT0000: update sequence from 0x100000000 to 0x280000bd0
Lustre: t32fs-OST0001-osc-MDT0000: update sequence from 0x100010000 to 0x2c0000bd0
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0001.failover.node=10.240.26.126@tcp
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0001.mdc.max_rpcs_in_flight=9
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0001.lov.stripesize=4M
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug=-1
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs.quota.mdt='ug'
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-MDT0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs.quota.ost='ug'
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
LustreError: 495124:0:(qsd_reint.c:635:qqi_reint_delayed()) t32fs-MDT0000: Delaying reintegration for qtype:0 until pending updates are flushed.
LustreError: 495124:0:(qsd_reint.c:635:qqi_reint_delayed()) Skipped 1 previous similar message
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-ldiskfs.t32fs-OST0000.quota_slave.enabled
Lustre: DEBUG MARKER: test -f /tmp/t32/list
Lustre: DEBUG MARKER: test -f /tmp/t32/list2
Lustre: DEBUG MARKER: cat /tmp/t32/list2
Lustre: DEBUG MARKER: lctl set_param -n os[cd]*.*MDT*.force_sync=1
Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n osd*.*.force_sync=1
Lustre: DEBUG MARKER: test -f /tmp/t32/sha1sums
Lustre: DEBUG MARKER: cat /tmp/t32/sha1sums
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param t32fs-MDT0000.mdc.max_rpcs_in_flight='14'
Lustre: Modifying parameter t32fs-MDT0000-mdc.mdc.max_rpcs_in_flight in log t32fs-client
Lustre: Skipped 9 previous similar messages
Lustre: DEBUG MARKER: umount -d /tmp/t32/mnt/mdt
Lustre: Failing over t32fs-MDT0000
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@00000000259ea336[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000002c27ee60
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000336d30c1osd-ldiskfs-object@00000000336d30c1(i:0000000084b01e47:12/203470252)[plain]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@00000000259ea336
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@00000000a32a7ee6[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000090620c15
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000085d0f2faosd-ldiskfs-object@0000000085d0f2fa(i:000000007b63899f:81/3644679279)[lfix]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@00000000a32a7ee6
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@0000000001d555cd[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000059f2470f
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000008a84a3b2osd-ldiskfs-object@000000008a84a3b2(i:00000000941383b4:80/3644679278)[plain]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@0000000001d555cd
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@00000000cd321c84[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000006d2be9f6
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000f3708c5cosd-ldiskfs-object@00000000f3708c5c(i:000000001cc6c443:83/3644679348)[plain]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@00000000cd321c84
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@000000002d42968e[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000091879a19
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000009156351cosd-ldiskfs-object@000000009156351c(i:000000004fa2186f:82/3644679314)[plain]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@000000002d42968e
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@00000000f707aefd[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000e334ecb1
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000090ec2cafosd-ldiskfs-object@0000000090ec2caf(i:00000000d6337964:26696/3644679280)[plain]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@00000000f707aefd
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) header@0000000037e578b9[0x4, 1, [0x200000006:0x1020000:0x0] hash exist]{
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....mdt@00000000ce74bd48mdt-object@0000000037e578b9( , writecount=0)
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....mdd@000000002a54558bmdd-object@000000002a54558b(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....lod@0000000036826313lod-object@0000000036826313
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000b0ac717aosd-ldiskfs-object@00000000b0ac717a(i:0000000052f8c647:96/3644679364)[lfix]
LustreError: 496111:0:(lu_object.c:998:lu_site_print()) } header@0000000037e578b9

LustreError: 830728:0:(lod_dev.c:2171:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff93eeae0dd000
LustreError: 830728:0:(lod_dev.c:2171:lod_device_free()) LBUG
CPU: 1 PID: 830728 Comm: umount Kdump: loaded Tainted: G OE --------- - - 4.18.0-513.18.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x252/0x260 [lod]
class_free_dev+0x2ef/0x600 [obdclass]
class_detach+0x285/0x2d0 [obdclass]
class_process_config+0x1761/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x471/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x311/0x6f0 [mdd]
mdt_stack_fini+0x2ad/0x6c0 [mdt]
mdt_device_fini+0xa73/0xf60 [mdt]
? lu_context_init+0xa5/0x1b0 [obdclass]
obd_precleanup+0x1e5/0x220 [obdclass]
class_cleanup+0x31e/0x900 [obdclass]
class_process_config+0x3ad/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x456/0x780 [obdclass]
server_put_super+0x7b6/0x1310 [ptlrpc]
? fsnotify_unmount_inodes+0x11c/0x1b0
? evict_inodes+0x160/0x1b0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x19c/0x1b0
entry_SYSCALL_64_after_hwframe+0x61/0xc6
RIP: 0033:0x7fd87d684e9b

Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d -f /mnt/lustre-mds1
Lustre: lustre-MDT0000: Not available for connect from 10.240.43.142@tcp (stopping)
Lustre: Skipped 12 previous similar messages
Lustre: lustre-MDT0000: Not available for connect from 10.240.43.146@tcp (stopping)
Lustre: lustre-MDT0000: Not available for connect from 0@lo (stopping)
Lustre: Skipped 3 previous similar messages
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@00000000d9c84628[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000008bd7d324
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000006ea66a98osd-ldiskfs-object@000000006ea66a98(i:000000000c5f4361:112/1827530138)[plain]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@00000000d9c84628
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@000000000215e2e2[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000003c831ae1
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000adf7febfosd-ldiskfs-object@00000000adf7febf(i:00000000779d59b0:31715/1921399276)[plain]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@000000000215e2e2
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@000000008345774b[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000abddd7ca
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000c706e593osd-ldiskfs-object@00000000c706e593(i:0000000018915fe3:181/2435293991)[plain]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@000000008345774b
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@000000008bed5939[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000586a08ea
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000cc9ab135osd-ldiskfs-object@00000000cc9ab135(i:000000009d4b261e:31713/4264881685)[plain]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@000000008bed5939
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@000000005f19ff33[0x4, 1, [0x200000006:0x2020000:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....mdt@0000000054b24fcbmdt-object@000000005f19ff33( , writecount=0)
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....mdd@000000009deaf244mdd-object@000000009deaf244(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....lod@000000001d4487f5lod-object@000000001d4487f5
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000009de693afosd-ldiskfs-object@000000009de693af(i:00000000a28c1196:195/2514979359)[lfix]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@000000005f19ff33
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@00000000d4fe34e5[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000004e57a54a
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000002ddfab82osd-ldiskfs-object@000000002ddfab82(i:00000000ad58ee69:113/3534180672)[lfix]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@00000000d4fe34e5
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) header@00000000f576473f[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000003ed4cf9
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000a0765882osd-ldiskfs-object@00000000a0765882(i:00000000fa74d276:147/344048415)[plain]
LustreError: 830728:0:(lu_object.c:998:lu_site_print()) } header@00000000f576473f

LustreError: 778785:0:(lod_dev.c:2171:lod_device_free()) ASSERTION( atomic_read(&lu->ld_ref) == 0 ) failed: lu is ffff8d2a6deaf000
LustreError: 778785:0:(lod_dev.c:2171:lod_device_free()) LBUG
CPU: 0 PID: 778785 Comm: umount Kdump: loaded Tainted: G OE --------- - - 4.18.0-513.18.1.el8_lustre.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
Call Trace:
dump_stack+0x41/0x60
lbug_with_loc.cold.8+0x5/0x43 [libcfs]
lod_device_free+0x252/0x260 [lod]
class_free_dev+0x2ef/0x600 [obdclass]
class_detach+0x285/0x2d0 [obdclass]
class_process_config+0x1761/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x471/0x780 [obdclass]
lod_obd_disconnect+0x100/0x1d0 [lod]
mdd_process_config+0x311/0x6f0 [mdd]
mdt_stack_fini+0x2ad/0x6c0 [mdt]
mdt_device_fini+0xa73/0xf60 [mdt]
obd_precleanup+0x1e5/0x220 [obdclass]
class_cleanup+0x31e/0x900 [obdclass]
class_process_config+0x3ad/0x21f0 [obdclass]
? class_manual_cleanup+0x191/0x780 [obdclass]
? __kmalloc+0x113/0x250
? lprocfs_counter_add+0x12a/0x1a0 [obdclass]
class_manual_cleanup+0x456/0x780 [obdclass]
server_put_super+0x7b6/0x1310 [ptlrpc]
? fsnotify_unmount_inodes+0x11c/0x1b0
? evict_inodes+0x160/0x1b0
generic_shutdown_super+0x6c/0x110
kill_anon_super+0x14/0x30
deactivate_locked_super+0x34/0x70
cleanup_mnt+0x3b/0x70
task_work_run+0x8a/0xb0
exit_to_usermode_loop+0xef/0x100
do_syscall_64+0x19c/0x1b0
entry_SYSCALL_64_after_hwframe+0x61/0xc6
RIP: 0033:0x7ff5203c5e9b

Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osc.*MDT*.sync_*
Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osp.*.destroys_in_flight
Lustre: DEBUG MARKER: lctl set_param fail_val=0 fail_loc=0
Lustre: DEBUG MARKER: lctl set_param -n os[cd]*.*MDT*.force_sync=1
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param lustre.quota.ost=none
Lustre: DEBUG MARKER: /usr/sbin/lctl conf_param lustre.quota.ost=ugp
LustreError: 778590:0:(qsd_reint.c:635:qqi_reint_delayed()) lustre-MDT0002: Delaying reintegration for qtype:0 until pending updates are flushed.
LustreError: 778590:0:(qsd_reint.c:635:qqi_reint_delayed()) Skipped 11 previous similar messages
Lustre: DEBUG MARKER: grep -c /mnt/lustre-mds1' ' /proc/mounts || true
Lustre: DEBUG MARKER: umount -d /mnt/lustre-mds1
Lustre: Failing over lustre-MDT0000
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000f436205f[0x4, 1, [0xa:0x0:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000002a1043b9
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000483979dcosd-ldiskfs-object@00000000483979dc(i:0000000088ec71c8:181/1015054879)[plain]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000f436205f
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000e6405032[0x4, 1, [0x200000003:0x0:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000efbf0ea9
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000020cdbbddosd-ldiskfs-object@0000000020cdbbdd(i:00000000ee343163:112/2416053815)[plain]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000e6405032
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000a055f66c[0x5, 1, [0x200000001:0x1017:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@000000000737744a
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000853221d9osd-ldiskfs-object@00000000853221d9(i:00000000f10699f0:31713/3338428518)[plain]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000a055f66c
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@000000006a53a969[0x4, 1, [0x200000006:0x1020000:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....mdt@00000000cb120c45mdt-object@000000006a53a969( , writecount=0)
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....mdd@00000000c0875c18mdd-object@00000000c0875c18(open_count=0, valid=0, cltime=0ns, flags=0)
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....lod@000000002e772365lod-object@000000002e772365
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000a77b5a63osd-ldiskfs-object@00000000a77b5a63(i:000000005c84fc02:194/2786538293)[lfix]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@000000006a53a969
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000e98d0a69[0x4, 1, [0x200000003:0x3:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000097a23202
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@000000000775e6e6osd-ldiskfs-object@000000000775e6e6(i:0000000069c9a05b:31715/3574367099)[plain]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000e98d0a69
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000424e13d8[0x4, 1, [0x1:0x0:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@0000000000e77e42
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@00000000075acd88osd-ldiskfs-object@00000000075acd88(i:00000000ce4bc3d8:147/989556685)[plain]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000424e13d8
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) header@00000000ca9b051e[0x4, 1, [0x200000003:0x2:0x0] hash exist]{
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....local_storage@00000000d0a27919
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) ....osd-ldiskfs@0000000045261630osd-ldiskfs-object@0000000045261630(i:0000000031708fd8:113/1183113377)[lfix]
LustreError: 778785:0:(lu_object.c:998:lu_site_print()) } header@00000000ca9b051e