general protection fault: 0000 [#1] SMP DEBUG_PAGEALLOC
Modules linked in: loop zfs(PO) zunicode(PO) zlua(PO) zcommon(PO) znvpair(PO) zavl(PO) icp(PO) spl(O) lustre(OE) osp(OE) ofd(OE) lod(OE) mdt(OE) mdd(OE) mgs(OE) osd_ldiskfs(OE) ldiskfs(OE) jbd2 mbcache lquota(OE) lfsck(OE) obdecho(OE) mgc(OE) mdc(OE) lov(OE) osc(OE) lmv(OE) fid(OE) fld(OE) ptlrpc_gss(OE) ptlrpc(OE) obdclass(OE) ksocklnd(OE) crc32_generic lnet(OE) dm_flakey dm_mod libcfs(OE) crc_t10dif crct10dif_generic crct10dif_common virtio_balloon pcspkr virtio_console i2c_piix4 ip_tables rpcsec_gss_krb5 ata_generic drm_kms_helper pata_acpi ttm drm drm_panel_orientation_quirks virtio_blk ata_piix serio_raw i2c_core floppy libata
CPU: 15 PID: 18054 Comm: mdt00_019 Kdump: loaded Tainted: P OE ------------ 3.10.0-7.9-debug #2
Hardware name: Red Hat KVM, BIOS 1.16.0-3.module_el8.7.0+1218+f626c2ff 04/01/2014
task: ffff8802d23549d0 ti: ffff8802d2378000 task.ti: ffff8802d2378000
RIP: 0010:[<ffffffffa0f16c33>] [<ffffffffa0f16c33>] lod_object_unlock+0x113/0x3c0 [lod]
RSP: 0018:ffff8802d237baa0 EFLAGS: 00010206
RAX: ffff8802cf24d708 RBX: 0000000000000002 RCX: dead000000000200
RDX: 0000000000000002 RSI: 5a5a5a5a5a5a5a5a RDI: ffff88025e832340
RBP: ffff8802d237bac8 R08: ffff880331400660 R09: 0000000000000000
R10: ffff880331400640 R11: ffff8802ad72f528 R12: ffff8802d223e588
R13: ffff8802d1c75f90 R14: ffff880290c7f748 R15: ffff88026492c968
FS: 0000000000000000(0000) GS:ffff880331dc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f88cd39b014 CR3: 0000000256222000 CR4: 00000000000007e0
Call Trace:
[<ffffffffa0d65b8d>] mdd_object_unlock+0x2d/0xd0 [mdd]
[<ffffffffa0deede2>] mdt_object_stripes_unlock+0x172/0x2d0 [mdt]
[<ffffffffa0df63ee>] mdt_reint_setattr+0xcbe/0x15f0 [mdt]
[<ffffffffa0df6da7>] mdt_reint_rec+0x87/0x240 [mdt]
[<ffffffffa0dcab3c>] mdt_reint_internal+0x74c/0xbc0 [mdt]
[<ffffffffa0dd2cd5>] ? mdt_thread_info_init+0xa5/0xc0 [mdt]
[<ffffffffa0dd59f7>] mdt_reint+0x67/0x150 [mdt]
[<ffffffffa071fcee>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<ffffffffa0664ab1>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<ffffffffa0666881>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<ffffffffa0665c20>] ? ptlrpc_wait_event+0x630/0x630 [ptlrpc]
[<ffffffff810ba114>] kthread+0xe4/0xf0
[<ffffffff810ba030>] ? kthread_create_on_node+0x140/0x140
[<ffffffff817ede5d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff810ba030>] ? kthread_create_on_node+0x140/0x140

LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x240000403:0x1:0x0]/14 failed: rc = -114
LustreError: 17911:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x200000403:0x9:0x0] mdc close failed: rc = -116
LustreError: 10860:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/6 failed: rc = -114
LustreError: 10860:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 1 previous similar message
LustreError: 18292:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x200000403:0x5:0x0] mdc close failed: rc = -116
LustreError: 18292:0:(file.c:247:ll_close_inode_openhandle()) Skipped 1 previous similar message
Lustre: 17689:0:(mdt_recovery.c:128:mdt_req_from_lrd()) @@@ restoring transno req@ffff8802cf0da840 x1824193959797888/t4294967556(0) o101->a18fdab1-ff48-40de-857a-f4cd57d115c3@0@lo:140/0 lens 376/864 e 0 to 0 dl 1739686995 ref 1 fl Interpret:H/202/0 rc 0/0 job:'dd.0' uid:0 gid:0
Lustre: 10861:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0002: opcode 2: before 512 < left 1278, rollback = 2
Lustre: 10861:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/1, destroy: 1/4/0
Lustre: 10861:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 4/4/0, xattr_set: 17/1278/0
Lustre: 10861:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 1/3/0
Lustre: 10861:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 17/335/2, delete: 3/6/0
Lustre: 10861:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 10/10/0, ref_del: 2/2/0
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 516 < left 618, rollback = 7
LustreError: 18414:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x280000403:0x1:0x0]/19 failed: rc = -114
LustreError: 18414:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 4 previous similar messages
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 505 < left 699, rollback = 2
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 44 previous similar messages
Lustre: 18414:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/6, destroy: 0/0/0
Lustre: 18414:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 46 previous similar messages
Lustre: 18414:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 9/699/0
Lustre: 18414:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 46 previous similar messages
Lustre: 18414:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 1/3/0
Lustre: 18414:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 46 previous similar messages
Lustre: 18414:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 11/239/4, delete: 0/0/0
Lustre: 18414:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 46 previous similar messages
Lustre: 18414:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 5/5/0, ref_del: 0/0/0
Lustre: 18414:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 46 previous similar messages
LustreError: 19575:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x240000403:0x41:0x0] mdc close failed: rc = -116
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 510 < left 1873, rollback = 2
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 45 previous similar messages
Lustre: 18414:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/2, destroy: 1/4/0
Lustre: 18414:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 45 previous similar messages
Lustre: 18414:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 22/1873/0
Lustre: 18414:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 45 previous similar messages
Lustre: 18414:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 1/3/0
Lustre: 18414:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 45 previous similar messages
Lustre: 18414:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 14/287/3, delete: 2/5/0
Lustre: 18414:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 45 previous similar messages
Lustre: 18414:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 7/7/0, ref_del: 1/1/0
Lustre: 18414:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 45 previous similar messages
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x2:0x0]/6 failed: rc = -114
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 5 previous similar messages
LustreError: 18326:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0002: failed to get lu_attr of [0x280000403:0x28:0x0]: rc = -2
LustreError: 20256:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0001: '8' migration was interrupted, run 'lfs migrate -m 2 -c 3 -H crush 8' to finish migration: rc = -1
LustreError: 18931:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '17' migration was interrupted, run 'lfs migrate -m 1 -c 2 -H crush 17' to finish migration: rc = -1
Lustre: 10861:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 500 < left 966, rollback = 2
Lustre: 10861:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 64 previous similar messages
Lustre: 10861:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/11, destroy: 0/0/0
Lustre: 10861:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 64 previous similar messages
Lustre: 10861:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 12/966/0
Lustre: 10861:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 64 previous similar messages
Lustre: 10861:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 4/54/0
Lustre: 10861:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 64 previous similar messages
Lustre: 10861:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 10/203/4, delete: 0/0/0
Lustre: 10861:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 64 previous similar messages
Lustre: 10861:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 4/4/0, ref_del: 0/0/0
Lustre: 10861:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 64 previous similar messages
Lustre: 17589:0:(mdt_recovery.c:128:mdt_req_from_lrd()) @@@ restoring transno req@ffff8802c8c51440 x1824193963683968/t4294969634(0) o101->7d162cb2-f2aa-4f1d-82ed-49c87dcb5924@0@lo:147/0 lens 376/816 e 0 to 0 dl 1739687002 ref 1 fl Interpret:H/202/0 rc 0/0 job:'dd.0' uid:0 gid:0
LustreError: 20460:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x240000404:0x35:0x0]/11 failed: rc = -114
LustreError: 20460:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 15 previous similar messages
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0003: opcode 7: before 516 < left 618, rollback = 7
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 1 previous similar message
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 98 previous similar messages
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 98 previous similar messages
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 2/618/0, punch: 0/0/0, quota 4/150/0
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 98 previous similar messages
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 98 previous similar messages
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 98 previous similar messages
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0003: opcode 7: before 516 < left 618, rollback = 7
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 5 previous similar messages
2[23416]: segfault at 8 ip 00007f6ce45ee7e8 sp 00007ffdcc6ec0f0 error 4 in ld-2.17.so[7f6ce45e3000+22000]
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 503 < left 1189, rollback = 2
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 98 previous similar messages
LustreError: 23156:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000403:0xd6:0x0]: rc = -5
LustreError: 23156:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
Lustre: dir [0x240000404:0xd2:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
LustreError: 23156:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000403:0x144:0x0]: rc = -5
LustreError: 23156:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 2 previous similar messages
LustreError: 23156:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 23156:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 2 previous similar messages
LustreError: 21286:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0002: failed to get lu_attr of [0x280000403:0xb7:0x0]: rc = -2
LustreError: 22517:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x280000403:0xb7:0x0] mdc close failed: rc = -2
LustreError: 22517:0:(file.c:247:ll_close_inode_openhandle()) Skipped 4 previous similar messages
LustreError: 20500:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x280000403:0x2d:0x0]/15 failed: rc = -2
LustreError: 20500:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 18 previous similar messages
LustreError: 10861:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '0' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H crush 0' to finish migration: rc = -1
Lustre: 18931:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/8, destroy: 0/0/0
Lustre: 18931:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 111 previous similar messages
Lustre: 18931:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 16/1322/0
Lustre: 18931:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 111 previous similar messages
Lustre: 18931:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 4/54/0
Lustre: 18931:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 111 previous similar messages
Lustre: 18931:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 11/239/4, delete: 0/0/0
Lustre: 18931:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 111 previous similar messages
Lustre: 18931:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 5/5/0, ref_del: 0/0/0
Lustre: 18931:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 111 previous similar messages
LustreError: 26538:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x280000404:0xc9:0x0] mdc close failed: rc = -116
LustreError: 26538:0:(file.c:247:ll_close_inode_openhandle()) Skipped 1 previous similar message
LustreError: 20256:0:(osd_handler.c:4580:osd_ref_del()) lustre-MDT0000: nlink == 0 on [0x200000404:0x7f:0x0], maybe an upgraded file? (LU-3915)
Lustre: 25957:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0002: opcode 2: before 515 < left 742, rollback = 2
Lustre: 25957:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 226 previous similar messages
13[28689]: segfault at 8 ip 00007f60297c07e8 sp 00007ffeb9eae360 error 4 in ld-2.17.so[7f60297b5000+22000]
LustreError: 28689:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x200000403:0x293:0x0] mdc close failed: rc = -13
Lustre: dir [0x280000403:0x122:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
LustreError: 10861:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '12' migration was interrupted, run 'lfs migrate -m 1 -c 3 -H crush 12' to finish migration: rc = -1
LustreError: 10861:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 1 previous similar message
LustreError: 18326:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0002: failed to get lu_attr of [0x280000404:0x30:0x0]: rc = -2
LustreError: 18326:0:(mdd_object.c:3901:mdd_close()) Skipped 1 previous similar message
Lustre: 12398:0:(mdt_recovery.c:128:mdt_req_from_lrd()) @@@ restoring transno req@ffff8802e59e6e40 x1824193972532608/t4294972243(0) o101->a18fdab1-ff48-40de-857a-f4cd57d115c3@0@lo:170/0 lens 376/840 e 0 to 0 dl 1739687025 ref 1 fl Interpret:H/202/0 rc 0/0 job:'dd.0' uid:0 gid:0
LustreError: 23208:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0001: [0x240000404:0xc3:0x0] migrate mdt count mismatch 3 != 1
LustreError: 30769:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000404:0x160:0x0]: rc = -5
LustreError: 30769:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 2 previous similar messages
LustreError: 30769:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 30769:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 2 previous similar messages
LustreError: 18377:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x280000403:0x14:0x0]/17 failed: rc = -114
LustreError: 18377:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 30 previous similar messages
13[31108]: segfault at 8 ip 00007f364d4a97e8 sp 00007fff38233260 error 4 in ld-2.17.so[7f364d49e000+22000]
LustreError: 20322:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0001: '4' migration was interrupted, run 'lfs migrate -m 0 -c 2 -H crush 4' to finish migration: rc = -1
Lustre: 20500:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/9, destroy: 0/0/0
Lustre: 20500:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 313 previous similar messages
Lustre: 20500:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 10/788/0
Lustre: 20500:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 313 previous similar messages
Lustre: 20500:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 4/54/0
Lustre: 20500:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 313 previous similar messages
Lustre: 20500:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 11/239/4, delete: 0/0/0
Lustre: 20500:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 313 previous similar messages
Lustre: 20500:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 5/5/0, ref_del: 0/0/0
Lustre: 20500:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 313 previous similar messages
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 506 < left 1144, rollback = 2
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 204 previous similar messages
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 516 < left 618, rollback = 7
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 7 previous similar messages
LustreError: 21286:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0000: failed to get lu_attr of [0x200000404:0x15c:0x0]: rc = -2
LustreError: 29490:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x200000404:0x15c:0x0] mdc close failed: rc = -2
LustreError: 29490:0:(file.c:247:ll_close_inode_openhandle()) Skipped 4 previous similar messages
Lustre: dir [0x240000403:0x12b:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0000: opcode 7: before 515 < left 618, rollback = 7
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 5 previous similar messages
LustreError: 17126:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0002: failed to get lu_attr of [0x280000404:0x262:0x0]: rc = -2
LustreError: 3171:0:(llite_lib.c:1846:ll_update_lsm_md()) lustre: [0x280000404:0x2a9:0x0] dir layout mismatch:
LustreError: 3171:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=2 count=2 index=2 hash=crush:0x2000003 max_inherit=0 max_inherit_rr=0 version=2 migrate_offset=0 migrate_hash=invalid:0 pool=
LustreError: 3171:0:(lustre_lmv.h:167:lmv_stripe_object_dump()) stripe[0] [0x280000400:0x18:0x0]
LustreError: 3171:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=1 count=4 index=2 hash=crush:0x82000003 max_inherit=0 max_inherit_rr=0 version=1 migrate_offset=2 migrate_hash=crush:2000003 pool=
LustreError: 20292:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '19' migration was interrupted, run 'lfs migrate -m 2 -c 2 -H crush 19' to finish migration: rc = -1
LustreError: 4187:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000403:0x1f9:0x0]: rc = -5
LustreError: 4187:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
Lustre: dir [0x200000404:0x2ff:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 10 previous similar messages
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0000: opcode 7: before 515 < left 528, rollback = 7
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 3 previous similar messages
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x240000404:0x301:0x0]/19 failed: rc = -114
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 41 previous similar messages
LustreError: 5025:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000403:0x507:0x0]: rc = -5
LustreError: 5025:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
Lustre: 16797:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x2ee:0x0] with magic=0xbd60bd0
Lustre: 24081:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x290:0x0] with magic=0xbd60bd0
Lustre: 24081:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 3 previous similar messages
Lustre: 10845:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 1/4/4, destroy: 0/0/0
Lustre: 10845:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 381 previous similar messages
Lustre: 10845:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 9/742/0
Lustre: 10845:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 381 previous similar messages
Lustre: 10845:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/18/0, punch: 0/0/0, quota 4/54/2
Lustre: 10845:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 381 previous similar messages
Lustre: 10845:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 2/33/1, delete: 0/0/0
Lustre: 10845:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 381 previous similar messages
Lustre: 10845:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 10845:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 381 previous similar messages
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 509 < left 816, rollback = 2
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 357 previous similar messages
LustreError: 4538:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x240000404:0x42c:0x0]: rc = -5
LustreError: 4538:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 4573:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x240000403:0x2bd:0x0] mdc close failed: rc = -116
LustreError: 4573:0:(file.c:247:ll_close_inode_openhandle()) Skipped 6 previous similar messages
LustreError: 24076:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0001: '13' migration was interrupted, run 'lfs migrate -m 1 -c 3 -H crush 13' to finish migration: rc = -1
Lustre: 20322:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x304:0x0] with magic=0xbd60bd0
Lustre: 20322:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 1 previous similar message
Lustre: 16849:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x2d8:0x0] with magic=0xbd60bd0
Lustre: 16849:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 1 previous similar message
Lustre: 17454:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0002-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x280000403:0x42a:0x0] with magic=0xbd60bd0
Lustre: 17454:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 27 previous similar messages
6[17068]: segfault at 0 ip 0000000000401d60 sp 00007ffc42ae72c8 error 6 in 6[400000+6000]
17[17263]: segfault at 8 ip 00007f1e0f5057e8 sp 00007ffe0b32ba60 error 4 in ld-2.17.so[7f1e0f4fa000+22000]
5[17446]: segfault at 8 ip 00007fe1469d57e8 sp 00007ffcac146940 error 4 in ld-2.17.so[7fe1469ca000+22000]
17[17306]: segfault at 8 ip 00007f31c52af7e8 sp 00007ffe88288090 error 4 in ld-2.17.so[7f31c52a4000+22000]
6[18785]: segfault at 406000 ip 0000000000406000 sp 00007ffd23ab64a8 error 14 in 6[606000+1000]
5[19067]: segfault at 8 ip 00007f5c5a7ba7e8 sp 00007ffc80b40c60 error 4 in ld-2.17.so[7f5c5a7af000+22000]
5[18863]: segfault at 8 ip 00007f63487c27e8 sp 00007ffc576b5110 error 4 in ld-2.17.so[7f63487b7000+22000]
Lustre: 17323:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x446:0x0] with magic=0xbd60bd0
Lustre: 17323:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 1 previous similar message
5[20052]: segfault at 8 ip 00007f5825b797e8 sp 00007ffc1bb6b460 error 4 in ld-2.17.so[7f5825b6e000+22000]
0[19871]: segfault at 0 ip 0000000000401d60 sp 00007ffed13579c8 error 6 in 0[400000+6000]
LustreError: 21633:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000403:0x5b8:0x0]: rc = -5
LustreError: 21633:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 8 previous similar messages
LustreError: 21633:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 21633:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 8 previous similar messages
6[24794]: segfault at 8 ip 00007fd1ba0eb7e8 sp 00007ffeb0c10970 error 4 in ld-2.17.so[7fd1ba0e0000+22000]
LustreError: 131:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep sleep [0x200000403:0x5b8:0x0] inode@0000000000000000: rc = -5
9[27421]: segfault at 8 ip 00007f251fa657e8 sp 00007fff0d7a5a50 error 4 in ld-2.17.so[7f251fa5a000+22000]
10[29525]: segfault at 0 ip 0000000000401d60 sp 00007ffcdc351838 error 6 in 10[400000+6000]
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x280000403:0x1:0x0]/15 failed: rc = -114
LustreError: 10861:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 31 previous similar messages
LustreError: 32327:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000403:0x5b8:0x0]: rc = -5
LustreError: 32327:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 12 previous similar messages
LustreError: 32327:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 32327:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 12 previous similar messages
LustreError: 18481:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x280000404:0x640:0x0] mdc close failed: rc = -116
LustreError: 18481:0:(file.c:247:ll_close_inode_openhandle()) Skipped 1 previous similar message
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 516 < left 522, rollback = 7
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 1 previous similar message
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 197 previous similar messages
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 197 previous similar messages
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 2/522/0, punch: 0/0/0, quota 4/150/0
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 197 previous similar messages
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 197 previous similar messages
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 197 previous similar messages
9[4418]: segfault at 406000 ip 0000000000406000 sp 00007ffd00d9f648 error 14 in 9[606000+1000]
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 512 < left 922, rollback = 2
Lustre: 20322:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 189 previous similar messages
Lustre: dir [0x200000404:0x8a8:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
5[7913]: segfault at 8 ip 00007f1db697c7e8 sp 00007ffec8953ff0 error 4 in ld-2.17.so[7f1db6971000+22000]
9[13940]: segfault at 8 ip 00007f278b0947e8 sp 00007ffcb2c43940 error 4 in ld-2.17.so[7f278b089000+22000]
Lustre: 23225:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0002-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x280000403:0x73c:0x0] with magic=0xbd60bd0
Lustre: 23225:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 1 previous similar message
13[16040]: segfault at 0 ip 0000000000403e5f sp 00007ffe6621f060 error 6 in 13[400000+6000]
11[18446]: segfault at 8 ip 00007fd2b35c67e8 sp 00007ffc8b524a60 error 4 in ld-2.17.so[7fd2b35bb000+22000]
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 516 < left 618, rollback = 7
Lustre: 18558:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 15 previous similar messages
5[21951]: segfault at 8 ip 00007f3f53eb57e8 sp 00007ffe5e31c8a0 error 4 in ld-2.17.so[7f3f53eaa000+22000]
17[25369]: segfault at 8 ip 00007f1bb02727e8 sp 00007ffc5d842bb0 error 4 in ld-2.17.so[7f1bb0267000+22000]
16[25754]: segfault at 8 ip 00007f7cb66457e8 sp 00007ffe66d14780 error 4 in ld-2.17.so[7f7cb663a000+22000]
Lustre: 17653:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000403:0x8bb:0x0] with magic=0xbd60bd0
Lustre: 17653:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 1 previous similar message
12[7129]: segfault at 8 ip 00007f813dbe07e8 sp 00007ffc46398560 error 4 in ld-2.17.so[7f813dbd5000+22000]
Lustre: 18504:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0002: opcode 7: before 516 < left 618, rollback = 7
Lustre: 18504:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 51 previous similar messages
LustreError: 19202:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000404:0x968:0x0]: rc = -2
7[11521]: segfault at 8 ip 00007f0ca75737e8 sp 00007fff518bb6f0 error 4 in ld-2.17.so[7f0ca7568000+22000]
Lustre: mdt00_027: service thread pid 23230 was inactive for 40.128 seconds. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
Pid: 23230, comm: mdt00_027 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_fini+0xa54/0xf50 [ptlrpc]
[<0>] ldlm_cli_enqueue+0x472/0xac0 [ptlrpc]
[<0>] osp_md_object_lock+0x151/0x2f0 [osp]
[<0>] lod_object_lock+0xdb/0x7d0 [lod]
[<0>] mdd_object_lock+0x2d/0xd0 [mdd]
[<0>] mdt_remote_object_lock_try+0x14c/0x189 [mdt]
[<0>] mdt_object_lock_internal+0x3c4/0x470 [mdt]
[<0>] mdt_object_lock+0x88/0x1c0 [mdt]
[<0>] mdt_reint_link+0x8ef/0xf10 [mdt]
[<0>] mdt_reint_rec+0x87/0x240 [mdt]
[<0>] mdt_reint_internal+0x74c/0xbc0 [mdt]
[<0>] mdt_reint+0x67/0x150 [mdt]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
Pid: 18054, comm: mdt00_019 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_local+0x2ea/0x800 [ptlrpc]
[<0>] mdt_object_lock_internal+0x1b3/0x470 [mdt]
[<0>] mdt_object_lock_try+0xa0/0x250 [mdt]
[<0>] mdt_getattr_name_lock+0x173b/0x2d00 [mdt]
[<0>] mdt_intent_getattr+0x2cc/0x4e0 [mdt]
[<0>] mdt_intent_opc.constprop.73+0x211/0xc60 [mdt]
[<0>] mdt_intent_policy+0x10f/0x460 [mdt]
[<0>] ldlm_lock_enqueue+0x397/0x980 [ptlrpc]
[<0>] ldlm_handle_enqueue+0x507/0x1870 [ptlrpc]
[<0>] tgt_enqueue+0x68/0x240 [ptlrpc]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
Lustre: mdt00_025: service thread pid 23208 was inactive for 40.011 seconds. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
Lustre: Skipped 1 previous similar message
Pid: 23208, comm: mdt00_025 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_local+0x2ea/0x800 [ptlrpc]
[<0>] mdt_object_lock_internal+0x1b3/0x470 [mdt]
[<0>] mdt_object_lock_try+0xa0/0x250 [mdt]
[<0>] mdt_getattr_name_lock+0x173b/0x2d00 [mdt]
[<0>] mdt_intent_getattr+0x2cc/0x4e0 [mdt]
[<0>] mdt_intent_opc.constprop.73+0x211/0xc60 [mdt]
[<0>] mdt_intent_policy+0x10f/0x460 [mdt]
[<0>] ldlm_lock_enqueue+0x397/0x980 [ptlrpc]
[<0>] ldlm_handle_enqueue+0x507/0x1870 [ptlrpc]
[<0>] tgt_enqueue+0x68/0x240 [ptlrpc]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
LustreError: 20473:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x280000403:0x1:0x0]/5 failed: rc = -114
LustreError: 20473:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 36 previous similar messages
Lustre: 18377:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/2, destroy: 1/4/0
Lustre: 18377:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 141 previous similar messages
Lustre: 18377:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 13/1072/0
Lustre: 18377:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 141 previous similar messages
Lustre: 18377:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 7/105/0
Lustre: 18377:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 141 previous similar messages
Lustre: 18377:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 15/323/3, delete: 2/5/0
Lustre: 18377:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 141 previous similar messages
Lustre: 18377:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 8/8/0, ref_del: 1/1/0
Lustre: 18377:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 141 previous similar messages
LustreError: 19304:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0000: failed to get lu_attr of [0x200000403:0xfae:0x0]: rc = -2
LustreError: 12116:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x200000403:0xfae:0x0] mdc close failed: rc = -2
LustreError: 12116:0:(file.c:247:ll_close_inode_openhandle()) Skipped 29 previous similar messages
17[14012]: segfault at 8 ip 00007ffae1d6a7e8 sp 00007ffcb0384d10 error 4 in ld-2.17.so[7ffae1d5f000+22000]
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 511 < left 788, rollback = 2
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 73 previous similar messages
13[21201]: segfault at 8 ip 00007fe9727567e8 sp 00007ffc6c033560 error 4 in ld-2.17.so[7fe97274b000+22000]
LustreError: 19830:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0001: '11' migration was interrupted, run 'lfs migrate -m 0 -c 1 -H crush 11' to finish migration: rc = -1
LustreError: 16610:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000404:0xb13:0x0]: rc = -5
LustreError: 16610:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
Lustre: dir [0x200000403:0x1004:0x0] stripe 1 readdir failed: -2, directory is partially accessed!
LustreError: 17808:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000404:0xad9:0x0]: rc = -2
LustreError: 17808:0:(mdd_object.c:3901:mdd_close()) Skipped 1 previous similar message
LustreError: 10612:0:(ldlm_lockd.c:252:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 0@lo ns: mdt-lustre-MDT0000_UUID lock: ffff8802b2646580/0x3e3ed41ae474a752 lrc: 3/0,0 mode: PR/PR res: [0x200000403:0xea0:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x60200400000020 nid: 0@lo remote: 0x3e3ed41ae474a71a expref: 862 pid: 16854 timeout: 421 lvb_type: 0
LustreError: lustre-MDT0000-mdc-ffff8802e82e5d28: operation mds_getxattr to node 0@lo failed: rc = -107
LustreError: 28893:0:(ldlm_lockd.c:2591:ldlm_cancel_handler()) ldlm_cancel from 0@lo arrived at 1739687314 with bad export cookie 4485255490871028808
Lustre: lustre-MDT0000-mdc-ffff8802e82e5d28: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0000-mdc-ffff8802e82e5d28: This client was evicted by lustre-MDT0000; in progress operations using this service will fail.
Lustre: mdt00_027: service thread pid 23230 completed after 100.309s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
LustreError: 25662:0:(llite_lib.c:1997:ll_md_setattr()) md_setattr fails: rc = -5
LustreError: Skipped 1 previous similar message
Lustre: mdt00_025: service thread pid 23208 completed after 99.715s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
LustreError: 2956:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x200000403:0xea0:0x0] error: rc = -108
Lustre: mdt00_019: service thread pid 18054 completed after 100.055s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: lustre-MDT0000-mdc-ffff8802e82e5d28: Connection restored to 192.168.123.22@tcp (at 0@lo)
Lustre: 16864:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000403:0xaea:0x0] with magic=0xbd60bd0
Lustre: 16864:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 3 previous similar messages
15[28438]: segfault at 8 ip 00007fbd42c8f7e8 sp 00007fff12197540 error 4 in ld-2.17.so[7fbd42c84000+22000]
Lustre: dir [0x280000404:0xc6b:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 19 previous similar messages
2[838]: segfault at 8 ip 00007faccfdeb7e8 sp 00007ffe264c22b0 error 4 in ld-2.17.so[7faccfde0000+22000]
Lustre: dir [0x200000404:0x132d:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 4 previous similar messages
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 515 < left 618, rollback = 7
Lustre: 13075:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 115 previous similar messages
4[9789]: segfault at ffffffff800e4707 ip 00000000004056bc sp 00007fff999ccb38 error 7 in 4[400000+6000]
LustreError: 24076:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '17' migration was interrupted, run 'lfs migrate -m 1 -c 3 -H crush 17' to finish migration: rc = -1
LustreError: 24076:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 6 previous similar messages
LustreError: 8954:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x280000403:0xe5d:0x0] : rc = -5
LustreError: 8954:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x280000403:0xe5d:0x0] error -5.
1[12688]: segfault at 1c ip 00000000004017b0 sp 00007ffd003e3370 error 6 in 1[400000+6000]
19[14625]: segfault at 406000 ip 0000000000406000 sp 00007ffc938705c8 error 14 in 19[606000+1000]
LustreError: 13971:0:(llite_nfs.c:426:ll_dir_get_parent_fid()) lustre: failure inode [0x280000404:0xf5f:0x0] get parent: rc = -116
3[16927]: segfault at 8 ip 00007f3b54f3a7e8 sp 00007ffcadd1a7f0 error 4 in ld-2.17.so[7f3b54f2f000+22000]
LustreError: 28209:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000404:0xedb:0x0]: rc = -2
LustreError: 28209:0:(mdd_object.c:3901:mdd_close()) Skipped 4 previous similar messages
14[19087]: segfault at 8 ip 00007f4eefd7b7e8 sp 00007fff7b9466f0 error 4 in ld-2.17.so[7f4eefd70000+22000]
LustreError: 20183:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x280000403:0xe5d:0x0] : rc = -5
LustreError: 270:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x280000403:0xe5d:0x0] : rc = -5
LustreError: 270:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep sleep [0x280000403:0xe5d:0x0] inode@0000000000000000: rc = -5
6[22097]: segfault at 8 ip 00007f872e3307e8 sp 00007ffe431e01c0 error 4 in ld-2.17.so[7f872e325000+22000]
traps: 0[23212] trap invalid opcode ip:404b86 sp:7ffe1cb6f9a8 error:0 in 0[400000+6000]
LustreError: 26906:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x240000404:0x108d:0x0]: rc = -5
LustreError: 26906:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 60 previous similar messages
LustreError: 26906:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 26906:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 60 previous similar messages
LustreError: 24752:0:(llite_lib.c:1846:ll_update_lsm_md()) lustre: [0x240000403:0x1221:0x0] dir layout mismatch:
LustreError: 24752:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=2 count=3 index=1 hash=crush:0x2000003 max_inherit=0 max_inherit_rr=0 version=2 migrate_offset=0 migrate_hash=invalid:0 pool=
LustreError: 24752:0:(lustre_lmv.h:167:lmv_stripe_object_dump()) stripe[0] [0x240000400:0x5d:0x0]
LustreError: 24752:0:(lustre_lmv.h:167:lmv_stripe_object_dump()) Skipped 5 previous similar messages
LustreError: 24752:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=1 count=4 index=1 hash=crush:0x82000003 max_inherit=0 max_inherit_rr=0 version=1 migrate_offset=3 migrate_hash=fnv_1a_64:2 pool=
Lustre: 23291:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000404:0x10be:0x0] with magic=0xbd60bd0
Lustre: 23291:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 41 previous similar messages
Lustre: dir [0x200000404:0x185b:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 1 previous similar message
8[32707]: segfault at 25607b1f ip 0000000000404ea6 sp 00007fff0f671320 error 4 in 8[400000+6000]
LustreError: 414:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 19 [0x240000403:0x1443:0x0] inode@0000000000000000: rc = -5
LustreError: 20460:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x240000403:0x1:0x0]/0 failed: rc = -114
LustreError: 20460:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 311 previous similar messages
3[17225]: segfault at 8 ip 00007f6392b497e8 sp 00007fff67884ad0 error 4 in ld-2.17.so[7f6392b3e000+22000]
Lustre: 13077:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 13077:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 3463 previous similar messages
Lustre: 13077:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0
Lustre: 13077:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 3463 previous similar messages
Lustre: 13077:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 2/618/0, punch: 0/0/0, quota 1/3/0
Lustre: 13077:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 3463 previous similar messages
Lustre: 13077:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0
Lustre: 13077:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 3463 previous similar messages
Lustre: 13077:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 13077:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 3463 previous similar messages
LustreError: 18594:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x280000403:0x1667:0x0] mdc close failed: rc = -116
LustreError: 18594:0:(file.c:247:ll_close_inode_openhandle()) Skipped 74 previous similar messages
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0001: opcode 2: before 505 < left 699, rollback = 2
Lustre: 18414:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 3257 previous similar messages
15[19239]: segfault at 8 ip 00007f1d097067e8 sp 00007ffc91fae6d0 error 4 in ld-2.17.so[7f1d096fb000+22000]
LustreError: 20292:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '9' migration was interrupted, run 'lfs migrate -m 1 -c 1 -H crush 9' to finish migration: rc = -1
LustreError: 20292:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 6 previous similar messages
LustreError: 21286:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000403:0x14bf:0x0]: rc = -2
LustreError: 21286:0:(mdd_object.c:3901:mdd_close()) Skipped 8 previous similar messages
17[26276]: segfault at 8 ip 00007fed9ef777e8 sp 00007ffc09ad77a0 error 4 in ld-2.17.so[7fed9ef6c000+22000]
LustreError: 396:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 11 [0x0:0x0:0x0] inode@0000000000000000: rc = -5
LustreError: 396:0:(statahead.c:825:ll_statahead_interpret_work()) Skipped 1 previous similar message
LustreError: 10612:0:(ldlm_lockd.c:252:expired_lock_main()) ### lock callback timer expired after 101s: evicting client at 0@lo ns: mdt-lustre-MDT0000_UUID lock: ffff8802adfbcf00/0x3e3ed41ae4b062b6 lrc: 3/0,0 mode: PR/PR res: [0x200000405:0x912:0x0].0x0 bits 0x13/0x0 rrc: 7 type: IBT gid 0 flags: 0x60200400000020 nid: 0@lo remote: 0x3e3ed41ae4b0629a expref: 1365 pid: 23300 timeout: 694 lvb_type: 0
LustreError: lustre-MDT0000-mdc-ffff8802e5d44138: operation mds_getxattr to node 0@lo failed: rc = -107
LustreError: Skipped 1 previous similar message
Lustre: lustre-MDT0000-mdc-ffff8802e5d44138: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0000-mdc-ffff8802e5d44138: This client was evicted by lustre-MDT0000; in progress operations using this service will fail.
LustreError: 5821:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x200000405:0x912:0x0] error: rc = -108
LustreError: 5821:0:(file.c:6007:ll_inode_revalidate_fini()) Skipped 48 previous similar messages
Lustre: dir [0x280000404:0x170b:0x0] stripe 1 readdir failed: -108, directory is partially accessed!
Lustre: Skipped 9 previous similar messages
LustreError: 17594:0:(client.c:1282:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@ffff8802c8c39e40 x1824194119908352/t0(0) o104->lustre-MDT0000@0@lo:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295
LustreError: 31321:0:(mdc_request.c:1457:mdc_read_page()) lustre-MDT0000-mdc-ffff8802e5d44138: [0x200000405:0xbe5:0x0] lock enqueue fails: rc = -108
LustreError: 32578:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x200000007:0x1:0x0] error: rc = -108
LustreError: 32578:0:(file.c:6007:ll_inode_revalidate_fini()) Skipped 357 previous similar messages
LustreError: 32147:0:(ldlm_resource.c:1149:ldlm_resource_complain()) lustre-MDT0000-mdc-ffff8802e5d44138: namespace resource [0x200000007:0x1:0x0].0x0 (ffff8802acd16fc0) refcount nonzero (1) after lock cleanup; forcing cleanup.
Lustre: lustre-MDT0000-mdc-ffff8802e5d44138: Connection restored to 192.168.123.22@tcp (at 0@lo)
16[1009]: segfault at 8 ip 00007f208f39b7e8 sp 00007ffe15130e30 error 4 in ld-2.17.so[7f208f390000+22000]
LustreError: 24:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 11 [0x0:0x0:0x0] inode@0000000000000000: rc = -5
14[3950]: segfault at 8 ip 00007f0d3665b7e8 sp 00007ffd591e0a80 error 4 in ld-2.17.so[7f0d36650000+22000]
LustreError: 28750:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e82e5d28: cannot apply new layout on [0x240000404:0x1464:0x0] : rc = -5
LustreError: 28750:0:(lov_object.c:1341:lov_layout_change()) Skipped 1 previous similar message
LustreError: 28750:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000404:0x1464:0x0] error -5.
LustreError: 6016:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e82e5d28: cannot apply new layout on [0x240000404:0x1464:0x0] : rc = -5
LustreError: 6016:0:(lov_object.c:1341:lov_layout_change()) Skipped 2 previous similar messages
7[5646]: segfault at 8 ip 00007fe873aa67e8 sp 00007ffe2ec1f3d0 error 4 in ld-2.17.so[7fe873a9b000+22000]
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0000: opcode 7: before 516 < left 618, rollback = 7
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 151 previous similar messages
LustreError: 16854:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0002: [0x280000404:0x1815:0x0] migrate mdt count mismatch 3 != 2
setfattr (10691) used greatest stack depth: 10080 bytes left
LustreError: 8395:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e82e5d28: cannot apply new layout on [0x240000404:0x1775:0x0] : rc = -5
LustreError: 8395:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000404:0x1775:0x0] error -5.
4[30482]: segfault at 0 ip 0000000000403cf0 sp 00007ffd065d1b78 error 6 in 4[400000+6000]
traps: 19[1141] general protection ip:4046bf sp:7ffc1c552ec0 error:0 in 19[400000+6000]
LustreError: 2510:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000406:0x124:0x0]: rc = -5
LustreError: 2510:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 71 previous similar messages
LustreError: 2510:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 2510:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 71 previous similar messages
Lustre: 10834:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0002-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x280000404:0x1b55:0x0] with magic=0xbd60bd0
Lustre: 10834:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 37 previous similar messages
16[8329]: segfault at 25002060 ip 0000000000404c95 sp 00007ffede5a0958 error 6 in 16[400000+6000]
16[8692]: segfault at 25202060 ip 0000000000404c95 sp 00007ffd6d766a68 error 6 in 16[400000+6000]
10[8685]: segfault at 1c ip 00000000004017b0 sp 00007ffc4a923e90 error 6 in 10[400000+6000]
LustreError: 1965:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 6 [0x200000406:0x6ee:0x0] inode@0000000000000000: rc = -5
LustreError: 1965:0:(statahead.c:825:ll_statahead_interpret_work()) Skipped 2 previous similar messages
LustreError: 20292:0:(lustre_lmv.h:500:lmv_is_sane()) unknown layout LMV: magic=0xcd40cd0 count=2 index=1 hash=crush:0x82000003 version=1 migrate_offset=1 migrate_hash=fnv_1a_64:2 pool=
19[15641]: segfault at 8 ip 00007ffa5599e7e8 sp 00007ffd13a84650 error 4 in ld-2.17.so[7ffa55993000+22000]
LustreError: 20500:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0001: '12' migration was interrupted, run 'lfs migrate -m 1 -c 3 -H crush 12' to finish migration: rc = -1
LustreError: 20500:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 16 previous similar messages
LustreError: 17808:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000404:0x1a82:0x0]: rc = -2
LustreError: 17808:0:(mdd_object.c:3901:mdd_close()) Skipped 11 previous similar messages
18[28864]: segfault at 8 ip 00007f242eb9b7e8 sp 00007fff0bcdbed0 error 4 in ld-2.17.so[7f242eb90000+22000]
11[30023]: segfault at 8 ip 00007f97c37cf7e8 sp 00007ffed55b64c0 error 4 in ld-2.17.so[7f97c37c4000+22000]
LustreError: 87:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 5 [0x0:0x0:0x0] inode@0000000000000000: rc = -5
LustreError: 87:0:(statahead.c:825:ll_statahead_interpret_work()) Skipped 1 previous similar message
LustreError: 313:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e82e5d28: cannot apply new layout on [0x200000406:0xbf7:0x0] : rc = -5
LustreError: 313:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x200000406:0xbf7:0x0] error -5.
LustreError: 32207:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x280000403:0x2042:0x0] error -5.
LustreError: 32207:0:(vvp_io.c:1903:vvp_io_init()) Skipped 1 previous similar message
Lustre: dir [0x240000403:0x20e0:0x0] stripe 1 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 21 previous similar messages
traps: 15[10158] general protection ip:404ba8 sp:7ffcf574dba8 error:0 in 15[400000+6000]
LustreError: 11053:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x240000403:0x1f71:0x0] : rc = -5
LustreError: 11053:0:(lov_object.c:1341:lov_layout_change()) Skipped 10 previous similar messages
17[11292]: segfault at 406000 ip 0000000000406000 sp 00007ffd60af18b8 error 14 in 17[606000+1000]
7[14792]: segfault at 8 ip 00007fc967ee57e8 sp 00007ffe20fe6e00 error 4 in ld-2.17.so[7fc967eda000+22000]
LustreError: 17200:0:(llite_lib.c:1997:ll_md_setattr()) md_setattr fails: rc = -5
5[18986]: segfault at 8 ip 00007f2e5b39b7e8 sp 00007ffec06d9b60 error 4 in ld-2.17.so[7f2e5b390000+22000]
4[19932]: segfault at 8 ip 00007fa3c4aaa7e8 sp 00007ffcbb32e130 error 4 in ld-2.17.so[7fa3c4a9f000+22000]
LustreError: 210:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 11 [0x240000403:0x216f:0x0] inode@0000000000000000: rc = -5
7[24404]: segfault at 8 ip 00007ff675bd37e8 sp 00007ffd6fbf6c60 error 4 in ld-2.17.so[7ff675bc8000+22000]
LustreError: 26579:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x240000403:0x1f71:0x0] : rc = -5
LustreError: 26579:0:(lov_object.c:1341:lov_layout_change()) Skipped 3 previous similar messages
LustreError: 27313:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x280000403:0x2674:0x0] error -5.
LustreError: 27313:0:(vvp_io.c:1903:vvp_io_init()) Skipped 1 previous similar message
19[3052]: segfault at 8 ip 00007fa26d6f57e8 sp 00007ffd7d6189c0 error 4 in ld-2.17.so[7fa26d6ea000+22000]
LustreError: 17689:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0002: [0x280000404:0x266d:0x0] migrate mdt count mismatch 2 != 3
LustreError: 17689:0:(mdt_xattr.c:402:mdt_dir_layout_update()) Skipped 1 previous similar message
LustreError: 31655:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000404:0x23fe:0x0] error -5.
19[8371]: segfault at 8 ip 00007f001f7f07e8 sp 00007ffd52464c60 error 4 in ld-2.17.so[7f001f7e5000+22000]
2[8617]: segfault at 0 ip (null) sp 00007ffc249aa5d8 error 14 in 2[400000+6000]
5[11145]: segfault at 20b3805b ip 0000000000405020 sp 00007fffbbbfbf98 error 6 in 5[400000+6000]
2[17293]: segfault at 20b3805b ip 0000000000405020 sp 00007ffdf287b398 error 6 in 2[400000+6000]
10[18550]: segfault at 8 ip 00007f3f77b9c7e8 sp 00007ffc4f70dc80 error 4 in ld-2.17.so[7f3f77b91000+22000]
10[18660]: segfault at 8 ip 00007ff7ddead7e8 sp 00007ffc7b7fde00 error 4 in ld-2.17.so[7ff7ddea2000+22000]
LustreError: 18931:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x280000404:0x27cd:0x0]/14 failed: rc = -116
LustreError: 18931:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 620 previous similar messages
Lustre: 10859:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 3/12/6, destroy: 0/0/0
Lustre: 10859:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 6416 previous similar messages
Lustre: 10859:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 2/2/0, xattr_set: 12/966/0
Lustre: 10859:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 6416 previous similar messages
Lustre: 10859:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 5/55/0, punch: 0/0/0, quota 4/54/0
Lustre: 10859:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 6416 previous similar messages
Lustre: 10859:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 12/275/3, delete: 0/0/0
Lustre: 10859:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 6416 previous similar messages
Lustre: 10859:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 6/6/0, ref_del: 0/0/0
Lustre: 10859:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 6416 previous similar messages
LustreError: 21409:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e5d44138: inode [0x240000403:0x2816:0x0] mdc close failed: rc = -13
LustreError: 21409:0:(file.c:247:ll_close_inode_openhandle()) Skipped 178 previous similar messages
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0000: opcode 2: before 507 < left 1439, rollback = 2
Lustre: 20460:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 6144 previous similar messages
13[24708]: segfault at 8 ip 00007f04ef63e7e8 sp 00007ffe0e47f880 error 4 in ld-2.17.so[7f04ef633000+22000]
18[1464]: segfault at 406000 ip 0000000000406000 sp 00007ffe8155ac18 error 14 in 18[606000+1000]
16[3135]: segfault at 0 ip (null) sp 00007ffca20ef5e8 error 14 in 16[400000+6000]
16[2175]: segfault at 0 ip (null) sp 00007ffdbd766468 error 14 in 16[400000+6000]
LustreError: 5823:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x280000403:0x2674:0x0] : rc = -5
LustreError: 5823:0:(lov_object.c:1341:lov_layout_change()) Skipped 6 previous similar messages
10[11033]: segfault at 0 ip 00000000004043f0 sp 00007fff4ddf8498 error 6 in 10[400000+6000]
11[15186]: segfault at 404520 ip 00000000004052cb sp 00007fff73817be8 error 7 in 11[400000+6000]
4[21184]: segfault at 8 ip 00007f789cf827e8 sp 00007ffdc2d19e60 error 4 in ld-2.17.so[7f789cf77000+22000]
5[22110]: segfault at 8 ip 00007fd3c14497e8 sp 00007fff98a14ad0 error 4 in ld-2.17.so[7fd3c143e000+22000]
5[22440]: segfault at 8 ip 00007f56076887e8 sp 00007ffdb4d43570 error 4 in ld-2.17.so[7f560767d000+22000]
9[24666]: segfault at 8 ip 00007f0e4b3817e8 sp 00007ffe0414df50 error 4 in ld-2.17.so[7f0e4b376000+22000]
9[25037]: segfault at 8 ip 00007f83051287e8 sp 00007ffce61ccc50 error 4 in ld-2.17.so[7f830511d000+22000]
0[25665]: segfault at 8 ip 00007f1e470707e8 sp 00007ffea149d960 error 4 in ld-2.17.so[7f1e47065000+22000]
Lustre: 26310:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0002: opcode 7: before 516 < left 618, rollback = 7
Lustre: 26310:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 305 previous similar messages
traps: 4[28322] general protection ip:4054fb sp:7ffd98cdd9c8 error:0 in 4[400000+6000]
15[28490]: segfault at 8 ip 00007f257b6c97e8 sp 00007ffc20318020 error 4 in ld-2.17.so[7f257b6be000+22000]
5[32182]: segfault at 8 ip 00007fdf5fa3f7e8 sp 00007ffc72ed1480 error 4 in ld-2.17.so[7fdf5fa34000+22000]
LustreError: 20130:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000403:0x2ff5:0x0] error -5.
LustreError: 20130:0:(vvp_io.c:1903:vvp_io_init()) Skipped 1 previous similar message
LustreError: 15676:0:(llite_lib.c:1846:ll_update_lsm_md()) lustre: [0x240000403:0x32e6:0x0] dir layout mismatch:
LustreError: 15676:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=2 count=2 index=1 hash=crush:0x2000003 max_inherit=0 max_inherit_rr=0 version=2 migrate_offset=0 migrate_hash=invalid:0 pool=
LustreError: 15676:0:(lustre_lmv.h:167:lmv_stripe_object_dump()) stripe[0] [0x240000400:0x109:0x0]
LustreError: 15676:0:(lustre_lmv.h:167:lmv_stripe_object_dump()) Skipped 6 previous similar messages
LustreError: 15676:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=1 count=3 index=1 hash=crush:0x82000003 max_inherit=0 max_inherit_rr=0 version=1 migrate_offset=2 migrate_hash=fnv_1a_64:2 pool=
LustreError: 19757:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=2 count=2 index=1 hash=crush:0x2000003 max_inherit=0 max_inherit_rr=0 version=2 migrate_offset=0 migrate_hash=invalid:0 pool=
LustreError: 19757:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd20cd0 refs=1 count=3 index=1 hash=crush:0x82000003 max_inherit=0 max_inherit_rr=0 version=1 migrate_offset=2 migrate_hash=fnv_1a_64:2 pool=
LustreError: 25074:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000404:0x330c:0x0]: rc = -5
LustreError: 25074:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 312 previous similar messages
LustreError: 25074:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 25074:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 312 previous similar messages
Lustre: 23293:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0000-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x200000406:0x2652:0x0] with magic=0xbd60bd0
Lustre: 23293:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 155 previous similar messages
8[28208]: segfault at 8 ip 00007f7dc1bc27e8 sp 00007ffe90bb8d80 error 4 in ld-2.17.so[7f7dc1bb7000+22000]
6[28616]: segfault at 8 ip 00007f683c7347e8 sp 00007ffddfb13c10 error 4 in ld-2.17.so[7f683c729000+22000]
2[29958]: segfault at 8 ip 00007fc7849dc7e8 sp 00007fff28a82e70 error 4 in ld-2.17.so[7fc7849d1000+22000]
LustreError: 18993:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep sleep [0x240000403:0x34dd:0x0] inode@0000000000000000: rc = -5
LustreError: 18993:0:(statahead.c:825:ll_statahead_interpret_work()) Skipped 4 previous similar messages
traps: 11[1992] trap invalid opcode ip:405d4c sp:7ffd34ac5968 error:0 in 11[400000+6000]
17[5566]: segfault at 8 ip 00007fd3a1c067e8 sp 00007ffe4a015930 error 4 in ld-2.17.so[7fd3a1bfb000+22000]
LustreError: 24081:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '11' migration was interrupted, run 'lfs migrate -m 2 -c 3 -H crush 11' to finish migration: rc = -1
LustreError: 24081:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 65 previous similar messages
4[12756]: segfault at 8 ip 00007fa13e7b57e8 sp 00007fffe341ebf0 error 4 in ld-2.17.so[7fa13e7aa000+22000]
13[11889]: segfault at 8 ip 00007fb790f127e8 sp 00007ffec5dc83e0 error 4 in ld-2.17.so[7fb790f07000+22000]
7[14119]: segfault at 8 ip 00007f95d02bf7e8 sp 00007fffe1489490 error 4 in ld-2.17.so[7f95d02b4000+22000]
LustreError: 1732:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0002: failed to get lu_attr of [0x280000404:0x33a3:0x0]: rc = -2
LustreError: 1732:0:(mdd_object.c:3901:mdd_close()) Skipped 24 previous similar messages
1[18071]: segfault at 8 ip 00007f589c6837e8 sp 00007ffd96badd10 error 4 in ld-2.17.so[7f589c678000+22000]
5[23285]: segfault at 8 ip 00007f0332b9c7e8 sp 00007ffe4a188830 error 4 in ld-2.17.so[7f0332b91000+22000]
LustreError: 13531:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0001: [0x240000404:0x373e:0x0] migrate mdt count mismatch 2 != 1
LustreError: 27110:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e82e5d28: cannot apply new layout on [0x240000403:0x36ea:0x0] : rc = -5
LustreError: 27110:0:(lov_object.c:1341:lov_layout_change()) Skipped 3 previous similar messages
LustreError: 27110:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000403:0x36ea:0x0] error -5.
6[32549]: segfault at 8 ip 00007f9c499537e8 sp 00007fff8477f440 error 4 in ld-2.17.so[7f9c49948000+22000]
6[1990]: segfault at 8 ip 00007f6766b597e8 sp 00007fff696b4c80 error 4 in ld-2.17.so[7f6766b4e000+22000]
Lustre: mdt00_007: service thread pid 16864 was inactive for 40.091 seconds. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:
Pid: 16864, comm: mdt00_007 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_local+0x2ea/0x800 [ptlrpc]
[<0>] mdt_object_lock_internal+0x1b3/0x470 [mdt]
[<0>] mdt_object_lock+0x88/0x1c0 [mdt]
[<0>] mdt_object_find_lock+0x54/0x170 [mdt]
[<0>] mdt_reint_setxattr+0x133/0x10e0 [mdt]
[<0>] mdt_reint_rec+0x87/0x240 [mdt]
[<0>] mdt_reint_internal+0x74c/0xbc0 [mdt]
[<0>] mdt_reint+0x67/0x150 [mdt]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
Pid: 13505, comm: mdt00_033 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_local+0x2ea/0x800 [ptlrpc]
[<0>] mdt_object_lock_internal+0x1b3/0x470 [mdt]
[<0>] mdt_object_lock_try+0xa0/0x250 [mdt]
[<0>] mdt_getattr_name_lock+0x173b/0x2d00 [mdt]
[<0>] mdt_intent_getattr+0x2cc/0x4e0 [mdt]
[<0>] mdt_intent_opc.constprop.73+0x211/0xc60 [mdt]
[<0>] mdt_intent_policy+0x10f/0x460 [mdt]
[<0>] ldlm_lock_enqueue+0x397/0x980 [ptlrpc]
[<0>] ldlm_handle_enqueue+0x507/0x1870 [ptlrpc]
[<0>] tgt_enqueue+0x68/0x240 [ptlrpc]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
Pid: 23300, comm: mdt00_032 3.10.0-7.9-debug #2 SMP Tue Feb 1 18:17:58 EST 2022
Call Trace:
[<0>] ldlm_completion_ast+0x913/0xd50 [ptlrpc]
[<0>] ldlm_cli_enqueue_local+0x2ea/0x800 [ptlrpc]
[<0>] mdt_object_lock_internal+0x1b3/0x470 [mdt]
[<0>] mdt_object_lock_try+0xa0/0x250 [mdt]
[<0>] mdt_getattr_name_lock+0x173b/0x2d00 [mdt]
[<0>] mdt_intent_getattr+0x2cc/0x4e0 [mdt]
[<0>] mdt_intent_opc.constprop.73+0x211/0xc60 [mdt]
[<0>] mdt_intent_policy+0x10f/0x460 [mdt]
[<0>] ldlm_lock_enqueue+0x397/0x980 [ptlrpc]
[<0>] ldlm_handle_enqueue+0x507/0x1870 [ptlrpc]
[<0>] tgt_enqueue+0x68/0x240 [ptlrpc]
[<0>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<0>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<0>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<0>] kthread+0xe4/0xf0
[<0>] ret_from_fork_nospec_begin+0x7/0x21
[<0>] 0xfffffffffffffffe
Lustre: mdt_io00_004: service thread pid 18414 was inactive for 72.041 seconds. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
Lustre: Skipped 1 previous similar message
Lustre: mdt_io00_000: service thread pid 10859 was inactive for 72.113 seconds. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
Lustre: Skipped 3 previous similar messages
Lustre: mdt_io00_016: service thread pid 24081 was inactive for 72.154 seconds. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
Lustre: Skipped 5 previous similar messages
Lustre: mdt_io00_003: service thread pid 18377 was inactive for 72.057 seconds. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one.
Lustre: Skipped 3 previous similar messages
LustreError: 10612:0:(ldlm_lockd.c:252:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 0@lo ns: mdt-lustre-MDT0000_UUID lock: ffff8802818e9300/0x3e3ed41ae584d5f3 lrc: 3/0,0 mode: PR/PR res: [0x200000406:0x2dcd:0x0].0x0 bits 0x13/0x0 rrc: 6 type: IBT gid 0 flags: 0x60200400000020 nid: 0@lo remote: 0x3e3ed41ae584d5c2 expref: 2653 pid: 17323 timeout: 1585 lvb_type: 0
LustreError: 23208:0:(ldlm_lockd.c:1492:ldlm_handle_enqueue()) ### lock on destroyed export ffff88029d6aae98 ns: mdt-lustre-MDT0000_UUID lock: ffff8802ce2ecf00/0x3e3ed41ae58d11cc lrc: 3/0,0 mode: PR/PR res: [0x200000406:0x2fbd:0x0].0x0 bits 0x1b/0x0 rrc: 17 type: IBT gid 0 flags: 0x50200000000000 nid: 0@lo remote: 0x3e3ed41ae58d11a9 expref: 2473 pid: 23208 timeout: 0 lvb_type: 0
LustreError: 11786:0:(ldlm_lockd.c:2591:ldlm_cancel_handler()) ldlm_cancel from 0@lo arrived at 1739688479 with bad export cookie 4485255490879918619
Lustre: lustre-MDT0000-mdc-ffff8802e5d44138: Connection to lustre-MDT0000 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0000-mdc-ffff8802e5d44138: operation ldlm_enqueue to node 0@lo failed: rc = -107
LustreError: lustre-MDT0000-mdc-ffff8802e5d44138: This client was evicted by lustre-MDT0000; in progress operations using this service will fail.
LustreError: 5387:0:(llite_lib.c:1997:ll_md_setattr()) md_setattr fails: rc = -108
LustreError: 5389:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x200000007:0x1:0x0] error: rc = -108
LustreError: 5389:0:(file.c:6007:ll_inode_revalidate_fini()) Skipped 10 previous similar messages
Lustre: mdt00_007: service thread pid 16864 completed after 100.566s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt00_033: service thread pid 13505 completed after 100.403s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt00_032: service thread pid 23300 completed after 100.381s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: lustre-MDT0000-mdc-ffff8802e5d44138: Connection restored to 192.168.123.22@tcp (at 0@lo)
LustreError: 10612:0:(ldlm_lockd.c:252:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 0@lo ns: mdt-lustre-MDT0001_UUID lock: ffff880298cc4f00/0x3e3ed41ae5870be9 lrc: 3/0,0 mode: PR/PR res: [0x240000404:0x3a2e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x60200400000020 nid: 0@lo remote: 0x3e3ed41ae5870ba3 expref: 984 pid: 10834 timeout: 1593 lvb_type: 0
Lustre: mdt_io00_004: service thread pid 18414 completed after 101.585s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt00_028: service thread pid 23233 completed after 100.375s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
LustreError: lustre-MDT0001-mdc-ffff8802e5d44138: operation mds_close to node 0@lo failed: rc = -107
Lustre: lustre-MDT0001-mdc-ffff8802e5d44138: Connection to lustre-MDT0001 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0001-mdc-ffff8802e5d44138: This client was evicted by lustre-MDT0001; in progress operations using this service will fail.
Lustre: 20460:0:(service.c:2350:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (101/1s); client may timeout req@ffff880085dcd040 x1824194366278272/t0(0) o36->7d162cb2-f2aa-4f1d-82ed-49c87dcb5924@0@lo:121/0 lens 688/456 e 2 to 0 dl 1739688486 ref 1 fl Complete:/200/0 rc -2/-2 job:'lfs.0' uid:0 gid:0
Lustre: mdt_io00_011: service thread pid 20460 completed after 101.574s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_010: service thread pid 20322 completed after 101.551s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
LustreError: 5844:0:(llite_lib.c:1997:ll_md_setattr()) md_setattr fails: rc = -108
LustreError: 5844:0:(llite_lib.c:1997:ll_md_setattr()) Skipped 1 previous similar message
LustreError: 6096:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x240000404:0x3ab1:0x0] error: rc = -108
LustreError: 5967:0:(mdc_request.c:1457:mdc_read_page()) lustre-MDT0001-mdc-ffff8802e5d44138: [0x240000404:0x3ab1:0x0] lock enqueue fails: rc = -108
LustreError: 5967:0:(mdc_request.c:1457:mdc_read_page()) Skipped 23 previous similar messages
LustreError: 6096:0:(file.c:6007:ll_inode_revalidate_fini()) Skipped 37 previous similar messages
Lustre: mdt_io00_019: service thread pid 7013 completed after 101.569s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_002: service thread pid 10861 completed after 101.539s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: lustre-MDT0001-mdc-ffff8802e5d44138: Connection restored to 192.168.123.22@tcp (at 0@lo)
Lustre: mdt_io00_000: service thread pid 10859 completed after 101.509s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_007: service thread pid 19873 completed after 101.590s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_001: service thread pid 10860 completed after 101.532s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_020: service thread pid 7234 completed after 101.500s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_012: service thread pid 20473 completed after 101.513s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_013: service thread pid 20500 completed after 101.154s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_005: service thread pid 18931 completed after 101.008s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_016: service thread pid 24081 completed after 101.037s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_015: service thread pid 24076 completed after 100.469s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_017: service thread pid 4225 completed after 98.562s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_003: service thread pid 18377 completed after 98.495s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
Lustre: mdt_io00_006: service thread pid 19830 completed after 97.580s. This likely indicates the system was overloaded (too many service threads, or not enough hardware resources).
4[10323]: segfault at 8 ip 00007fdc177467e8 sp 00007ffd5bf51aa0 error 4 in ld-2.17.so[7fdc1773b000+22000]
4[12915]: segfault at 1c ip 00000000004017b0 sp 00007ffcc38ed640 error 6 in 4[400000+6000]
Lustre: lustre-OST0000-osc-ffff8802e82e5d28: disconnect after 20s idle
INFO: task mrename:1935 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mrename D ffff8802db22cfc8 11744 1935 16300 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81252061>] lock_rename+0x31/0xe0
[<ffffffff812586af>] SYSC_renameat2+0x22f/0x570
[<ffffffff811ed462>] ? handle_mm_fault+0xc2/0x150
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff812597de>] SyS_renameat2+0xe/0x10
[<ffffffff8125981e>] SyS_rename+0x1e/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task rm:2448 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
rm D ffff880295646238 12600 2448 16338 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff8125830b>] do_unlinkat+0x13b/0x2b0
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff8125937b>] SyS_unlinkat+0x1b/0x40
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task file_exec.sh:2985 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
file_exec.sh D ffff88029b74bd58 12648 2985 16728 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81255126>] do_last+0x296/0x1280
[<ffffffff8125475e>] ? link_path_walk+0x27e/0x8c0
[<ffffffff8125712d>] path_openat+0xcd/0x5b0
[<ffffffff810c18ea>] ? prepare_creds+0x3a/0x200
[<ffffffff8121db0a>] ? ___slab_alloc+0x17a/0x560
[<ffffffff81258e7d>] do_filp_open+0x4d/0xb0
[<ffffffff810c18ea>] ? prepare_creds+0x3a/0x200
[<ffffffff810c15b2>] ? creds_are_invalid.part.1+0x12/0x50
[<ffffffff810c15b2>] ? creds_are_invalid.part.1+0x12/0x50
[<ffffffff810c1a40>] ? prepare_creds+0x190/0x200
[<ffffffff8124db84>] do_open_exec+0x24/0x110
[<ffffffff8124e1f1>] do_execve_common.isra.22+0x221/0x850
[<ffffffff81257c00>] ? getname_flags+0x30/0x1a0
[<ffffffff8124eac9>] SyS_execve+0x29/0x30
[<ffffffff817ee5a8>] stub_execve+0x48/0x80
LustreError: 10612:0:(ldlm_lockd.c:252:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 0@lo ns: mdt-lustre-MDT0002_UUID lock: ffff8802817a9300/0x3e3ed41ae58e4715 lrc: 3/0,0 mode: PR/PR res: [0x280000404:0x39f8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x60200400000020 nid: 0@lo remote: 0x3e3ed41ae58e46f9 expref: 829 pid: 23250 timeout: 1698 lvb_type: 0
LustreError: lustre-MDT0002-mdc-ffff8802e5d44138: operation mds_close to node 0@lo failed: rc = -107
Lustre: lustre-MDT0002-mdc-ffff8802e5d44138: Connection to lustre-MDT0002 (at 0@lo) was lost; in progress operations using this service will wait for recovery to complete
LustreError: lustre-MDT0002-mdc-ffff8802e5d44138: This client was evicted by lustre-MDT0002; in progress operations using this service will fail.
Lustre: dir [0x240000403:0x380d:0x0] stripe 1 readdir failed: -108, directory is partially accessed!
Lustre: Skipped 57 previous similar messages
LustreError: 2371:0:(mdc_request.c:1457:mdc_read_page()) lustre-MDT0002-mdc-ffff8802e5d44138: [0x280000403:0x1:0x0] lock enqueue fails: rc = -108
LustreError: 2371:0:(mdc_request.c:1457:mdc_read_page()) Skipped 19 previous similar messages
LustreError: 6900:0:(statahead.c:1801:is_first_dirent()) lustre: reading dir [0x280000403:0x1:0x0] at 0 stat_pid = 1898 : rc = -108
LustreError: 6900:0:(file.c:6007:ll_inode_revalidate_fini()) lustre: revalidate FID [0x280000404:0x39f8:0x0] error: rc = -108
Lustre: lustre-MDT0002-mdc-ffff8802e5d44138: Connection restored to 192.168.123.22@tcp (at 0@lo)
15[21045]: segfault at 8 ip 00007f98590f87e8 sp 00007fffd0d533f0 error 4 in ld-2.17.so[7f98590ed000+22000]
LustreError: 25932:0:(out_handler.c:921:out_tx_end()) lustre-MDT0000-osd: undo for /home/green/git/lustre-release/lustre/ptlrpc/../../lustre/target/out_handler.c:670: rc = -524
17[23228]: segfault at 406000 ip 0000000000406000 sp 00007ffe29c8f488 error 14 in 17[606000+1000]
8[23257]: segfault at 8 ip 00007fbffe8557e8 sp 00007ffec0b73710 error 4 in ld-2.17.so[7fbffe84a000+22000]
8[23763]: segfault at 8 ip 00007f595d07a7e8 sp 00007ffd17753380 error 4 in ld-2.17.so[7f595d06f000+22000]
LustreError: 23623:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x240000403:0x3a32:0x0] error -5.
LustreError: 23623:0:(vvp_io.c:1903:vvp_io_init()) Skipped 1 previous similar message
14[26798]: segfault at 8 ip 00007f192beef7e8 sp 00007fff4140e7a0 error 4 in ld-2.17.so[7f192bee4000+22000]
LustreError: 6080:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x2:0x0]/17 failed: rc = -114
LustreError: 6080:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 555 previous similar messages
Lustre: 8346:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 8346:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 6551 previous similar messages
Lustre: 8346:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0
Lustre: 8346:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 6550 previous similar messages
Lustre: 8346:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 2/564/0, punch: 0/0/0, quota 4/150/0
Lustre: 8346:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 6551 previous similar messages
Lustre: 8346:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0
Lustre: 8346:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 6551 previous similar messages
Lustre: 8346:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 8346:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 6550 previous similar messages
LustreError: 24622:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x280000405:0xc:0x0] mdc close failed: rc = -116
LustreError: 24622:0:(file.c:247:ll_close_inode_openhandle()) Skipped 247 previous similar messages
Lustre: 6080:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0002: opcode 2: before 505 < left 877, rollback = 2
Lustre: 6080:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 6243 previous similar messages
LustreError: 16797:0:(mdd_object.c:391:mdd_xattr_get()) lustre-MDD0002: object [0x280000405:0x1ed:0x0] not found: rc = -2
15[3474]: segfault at 8 ip 00007f97227de7e8 sp 00007ffdc2b570b0 error 4 in ld-2.17.so[7f97227d3000+22000]
traps: 1[8507] trap invalid opcode ip:40528a sp:7ffee00ecde8 error:0 in 1[400000+6000]
8[9676]: segfault at 8 ip 00007fe0898a97e8 sp 00007ffd50986f10 error 4 in ld-2.17.so[7fe08989e000+22000]
18[12854]: segfault at 8 ip 00007f4aa016c7e8 sp 00007ffd1ec953f0 error 4 in ld-2.17.so[7f4aa0161000+22000]
2[12827]: segfault at 8 ip 00007f318478e7e8 sp 00007ffc18563a00 error 4 in ld-2.17.so[7f3184783000+22000]
LustreError: 954:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 19 [0x240000403:0x3b59:0x0] inode@0000000000000000: rc = -5
LustreError: 18147:0:(mdt_open.c:1689:mdt_reint_open()) lustre-MDT0000: name '16' present, but FID [0x200000405:0x3b15:0x0] is invalid
LustreError: 23233:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0001: [0x240000405:0x10cb:0x0] migrate mdt count mismatch 3 != 1
12[20567]: segfault at 8 ip 00007fc8aa3707e8 sp 00007ffe3568c780 error 4 in ld-2.17.so[7fc8aa365000+22000]
Lustre: 18174:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0003: opcode 7: before 516 < left 540, rollback = 7
Lustre: 18174:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 327 previous similar messages
11[30378]: segfault at 8 ip 00007f1817f267e8 sp 00007fff8b913610 error 4 in ld-2.17.so[7f1817f1b000+22000]
11[29898]: segfault at 8 ip 00007f48b5ad37e8 sp 00007ffc1e5115b0 error 4 in ld-2.17.so[7f48b5ac8000+22000]
LustreError: 23208:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0002: [0x280000405:0x746:0x0] migrate mdt count mismatch 2 != 3
0[4026]: segfault at 8 ip 00007f59b0e527e8 sp 00007ffebedafb20 error 4 in ld-2.17.so[7f59b0e47000+22000]
1[4078]: segfault at 8 ip 00007fc17b2297e8 sp 00007ffc82a60810 error 4 in ld-2.17.so[7fc17b21e000+22000]
1[5217]: segfault at 8 ip 00007fc43917f7e8 sp 00007fff003e09f0 error 4 in ld-2.17.so[7fc439174000+22000]
LustreError: 12177:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0000: [0x200000407:0x17a5:0x0] migrate mdt count mismatch 2 != 3
10[5329]: segfault at 8 ip 00007fe97ab887e8 sp 00007ffc32e95260 error 4 in ld-2.17.so[7fe97ab7d000+22000]
LustreError: 24316:0:(out_lib.c:611:out_tx_attr_set_undo()) lustre-MDT0002-osd: attr set undo [0x280000401:0xef:0x0] unimplemented yet!: rc = -524
16[10178]: segfault at fffffffe ip 0000000000401610 sp 00007fff76233350 error 6 in 16[400000+6000]
3[11348]: segfault at 8 ip 00007ff426b007e8 sp 00007ffccfe4a460 error 4 in ld-2.17.so[7ff426af5000+22000]
18[14300]: segfault at 8 ip 00007f4eb7e0b7e8 sp 00007ffe0292d0c0 error 4 in ld-2.17.so[7f4eb7e00000+22000]
1[14668]: segfault at 8 ip 00007f6fbdb1c7e8 sp 00007ffdd493bf60 error 4 in ld-2.17.so[7f6fbdb11000+22000]
LustreError: 16040:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x240000403:0x456a:0x0]: rc = -5
LustreError: 16040:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 178 previous similar messages
LustreError: 16040:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 16040:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 178 previous similar messages
14[22331]: segfault at 8 ip 00007fca842b37e8 sp 00007ffc16534e90 error 4 in ld-2.17.so[7fca842a8000+22000]
15[23602]: segfault at 8 ip 00007f99e9b3f7e8 sp 00007ffc588ab990 error 4 in ld-2.17.so[7f99e9b34000+22000]
Lustre: 18147:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000403:0x456a:0x0] with magic=0xbd60bd0
Lustre: 18147:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 159 previous similar messages
LustreError: 23225:0:(mdt_open.c:1689:mdt_reint_open()) lustre-MDT0000: name '16' present, but FID [0x200000405:0x3b15:0x0] is invalid
14[1222]: segfault at 8 ip 00007f79046897e8 sp 00007fff1f651820 error 4 in ld-2.17.so[7f790467e000+22000]
LustreError: 20473:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '12' migration was interrupted, run 'lfs migrate -m 2 -c 2 -H crush 12' to finish migration: rc = -1
LustreError: 20473:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) Skipped 59 previous similar messages
LustreError: 2875:0:(lov_object.c:1341:lov_layout_change()) lustre-clilov-ffff8802e5d44138: cannot apply new layout on [0x280000403:0x4bb5:0x0] : rc = -5
LustreError: 2875:0:(lov_object.c:1341:lov_layout_change()) Skipped 13 previous similar messages
LustreError: 2875:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x280000403:0x4bb5:0x0] error -5.
LustreError: 2875:0:(vvp_io.c:1903:vvp_io_init()) Skipped 2 previous similar messages
2[22293]: segfault at 8 ip 00007f9109e8f7e8 sp 00007ffc6832e4f0 error 4 in ld-2.17.so[7f9109e84000+22000]
13[22541]: segfault at 8 ip 00007f3f17a477e8 sp 00007ffeff77f8b0 error 4 in ld-2.17.so[7f3f17a3c000+22000]
1[22744]: segfault at 8 ip 00007f695aba47e8 sp 00007ffdfa4d2340 error 4 in ld-2.17.so[7f695ab99000+22000]
7[30634]: segfault at 8 ip 00007fa5c546e7e8 sp 00007ffe65bcbc70 error 4 in ld-2.17.so[7fa5c5463000+22000]
18[31114]: segfault at 0 ip 0000000000403cf0 sp 00007ffcda367be8 error 6 in 18[400000+6000]
13[24004]: segfault at 8 ip 00007f5e4bcf07e8 sp 00007ffe566a6810 error 4 in ld-2.17.so[7f5e4bce5000+22000]
LustreError: 19202:0:(mdd_object.c:3901:mdd_close()) lustre-MDD0001: failed to get lu_attr of [0x240000405:0x1ef3:0x0]: rc = -2
LustreError: 19202:0:(mdd_object.c:3901:mdd_close()) Skipped 33 previous similar messages
LustreError: 10835:0:(mdt_open.c:1689:mdt_reint_open()) lustre-MDT0000: name '16' present, but FID [0x200000405:0x3b15:0x0] is invalid
9[31869]: segfault at 8 ip 00007fcd6b47c7e8 sp 00007fff84a6ef40 error 4 in ld-2.17.so[7fcd6b471000+22000]
9[31445]: segfault at 8 ip 00007efc5bb777e8 sp 00007ffe85db43e0 error 4 in ld-2.17.so[7efc5bb6c000+22000]
16[31636]: segfault at 0 ip (null) sp 00007ffd98d315a8 error 14 in 16[400000+6000]
16[32016]: segfault at 0 ip (null) sp 00007ffc3bfc03a8 error 14 in 16[400000+6000]
14[363]: segfault at 8 ip 00007ff3f8ce47e8 sp 00007fff85a18980 error 4 in ld-2.17.so[7ff3f8cd9000+22000]
3[1631]: segfault at 8 ip 00007f2df041d7e8 sp 00007ffe75545a30 error 4 in ld-2.17.so[7f2df0412000+22000]
3[17835]: segfault at 0 ip 00000000004043f0 sp 00007fff04846f08 error 6 in 8[400000+6000]
14[26694]: segfault at 0 ip 0000000000403e5f sp 00007ffef120f540 error 6 in 14[400000+6000]
6[29954]: segfault at 8 ip 00007f2b094017e8 sp 00007ffdea7c6de0 error 4 in ld-2.17.so[7f2b093f6000+22000]
LustreError: 4225:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/2 failed: rc = -114
LustreError: 4225:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 503 previous similar messages
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0
Lustre: 13075:0:(osd_handler.c:1967:osd_trans_dump_creds()) Skipped 5532 previous similar messages
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0
Lustre: 13075:0:(osd_handler.c:1974:osd_trans_dump_creds()) Skipped 5532 previous similar messages
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) write: 2/618/0, punch: 0/0/0, quota 1/3/0
Lustre: 13075:0:(osd_handler.c:1984:osd_trans_dump_creds()) Skipped 5532 previous similar messages
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0
Lustre: 13075:0:(osd_handler.c:1991:osd_trans_dump_creds()) Skipped 5532 previous similar messages
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0
Lustre: 13075:0:(osd_handler.c:1998:osd_trans_dump_creds()) Skipped 5532 previous similar messages
0[1665]: segfault at 8 ip 00007f91d89107e8 sp 00007ffcc87d8e10 error 4 in ld-2.17.so[7f91d8905000+22000]
3[4469]: segfault at 0 ip 0000000000403e5f sp 00007ffd789447f0 error 6 in 3[400000+6000]
11[8019]: segfault at 8 ip 00007f05c989e7e8 sp 00007ffc38da8b30 error 4 in ld-2.17.so[7f05c9893000+22000]
LustreError: 8768:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802e82e5d28: inode [0x200000407:0x2fe4:0x0] mdc close failed: rc = -116
LustreError: 8768:0:(file.c:247:ll_close_inode_openhandle()) Skipped 104 previous similar messages
Lustre: 6080:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-MDT0002: opcode 2: before 509 < left 983, rollback = 2
Lustre: 6080:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 4976 previous similar messages
6[16054]: segfault at 1c ip 00000000004017b0 sp 00007ffeb43752b0 error 6 in 6[400000+6000]
INFO: task mkdir:5569 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff88026d3cb760 11504 5569 16580 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task mkdir:9026 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff8802ab8dbd58 11616 9026 16387 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task mkdir:9027 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff880088068608 11616 9027 16336 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task mkdir:9103 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff880296f43d58 11616 9103 16483 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task mkdir:9872 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff8802afbce238 13056 9872 16625 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff810dbbab>] ? put_prev_entity+0x8b/0x400
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
INFO: task mkdir:10133 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mkdir D ffff8802d106ca80 13056 10133 16460 0x00000080
Call Trace:
[<ffffffff817e19d9>] schedule_preempt_disabled+0x39/0x90
[<ffffffff817df7ea>] __mutex_lock_slowpath+0x13a/0x340
[<ffffffff817dfa1d>] mutex_lock+0x2d/0x40
[<ffffffff81256f65>] filename_create+0x85/0x180
[<ffffffff81257c94>] ? getname_flags+0xc4/0x1a0
[<ffffffff81257f41>] user_path_create+0x41/0x60
[<ffffffff81259286>] SyS_mkdirat+0x46/0xe0
[<ffffffff817edf49>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff81259339>] SyS_mkdir+0x19/0x20
[<ffffffff817ee00c>] system_call_fastpath+0x1f/0x24
[<ffffffff817edf55>] ? system_call_after_swapgs+0xa2/0x13a
LustreError: 10843:0:(out_handler.c:921:out_tx_end()) lustre-MDT0002-osd: undo for /home/green/git/lustre-release/lustre/ptlrpc/../../lustre/target/out_handler.c:425: rc = -524
LustreError: 10843:0:(out_lib.c:1167:out_tx_index_delete_undo()) lustre-MDT0002-osd: Oops, can not rollback index_delete yet: rc = -524
LustreError: 617:0:(statahead.c:825:ll_statahead_interpret_work()) lustre: failed to prep 14 [0x0:0x0:0x0] inode@0000000000000000: rc = -1
0[32165]: segfault at 8 ip 00007efe6f7087e8 sp 00007ffc384f34e0 error 4 in ld-2.17.so[7efe6f6fd000+22000]
13[3652]: segfault at 0 ip (null) sp 00007ffce99443a8 error 14 in 13[400000+6000]
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) lustre-OST0001: opcode 7: before 515 < left 618, rollback = 7
Lustre: 24568:0:(osd_internal.h:1326:osd_trans_exec_op()) Skipped 611 previous similar messages
8[8706]: segfault at 8 ip 00007fdbf25c27e8 sp 00007ffeab9a0820 error 4 in ld-2.17.so[7fdbf25b7000+22000]
Lustre: dir [0x200000407:0x34f9:0x0] stripe 1 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 59 previous similar messages
15[13816]: segfault at 8 ip 00007f0298d1e7e8 sp 00007ffcb7210ac0 error 4 in ld-2.17.so[7f0298d13000+22000]
17[20733]: segfault at 8 ip 00007f10224277e8 sp 00007fff1814f390 error 4 in ld-2.17.so[7f102241c000+22000]
17[23579]: segfault at 8 ip 00007f707113b7e8 sp 00007ffc2af5aad0 error 4 in ld-2.17.so[7f7071130000+22000]
LustreError: 25431:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000405:0x66ea:0x0]: rc = -5
LustreError: 25431:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 52 previous similar messages
LustreError: 25431:0:(llite_lib.c:3712:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 25431:0:(llite_lib.c:3712:ll_prep_inode()) Skipped 52 previous similar messages
LustreError: 16854:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0000: [0x200000407:0x3794:0x0] migrate mdt count mismatch 2 != 3
LustreError: 27254:0:(vvp_io.c:1903:vvp_io_init()) lustre: refresh file layout [0x280000405:0x1e1f:0x0] error -5.
LustreError: 27254:0:(vvp_io.c:1903:vvp_io_init()) Skipped 1 previous similar message
12[482]: segfault at 8 ip 00007fbeda8457e8 sp 00007fffba93a100 error 4 in ld-2.17.so[7fbeda83a000+22000]
Lustre: 23250:0:(lod_lov.c:1418:lod_parse_striping()) lustre-MDT0001-mdtlov: EXTENSION flags=40 set on component[2]=1 of non-SEL file [0x240000403:0x589c:0x0] with magic=0xbd60bd0
Lustre: 23250:0:(lod_lov.c:1418:lod_parse_striping()) Skipped 55 previous similar messages

general protection fault: 0000 [#1] SMP DEBUG_PAGEALLOC
Modules linked in: lustre(OE) osp(OE) ofd(OE) lod(OE) mdt(OE) mdd(OE) mgs(OE) osd_zfs(OE) zfs(PO) zunicode(PO) zlua(PO) zcommon(PO) znvpair(PO) zavl(PO) icp(PO) spl(O) lquota(OE) lfsck(OE) obdecho(OE) mgc(OE) mdc(OE) lov(OE) osc(OE) lmv(OE) fid(OE) fld(OE) ptlrpc_gss(OE) ptlrpc(OE) obdclass(OE) ksocklnd(OE) crc32_generic lnet(OE) libcfs(OE) crc_t10dif crct10dif_generic crct10dif_common virtio_balloon virtio_console pcspkr i2c_piix4 ip_tables rpcsec_gss_krb5 ata_generic pata_acpi drm_kms_helper ttm drm drm_panel_orientation_quirks ata_piix serio_raw libata virtio_blk i2c_core floppy
CPU: 0 PID: 28982 Comm: mdt00_041 Kdump: loaded Tainted: P OE ------------ 3.10.0-7.9-debug #2
Hardware name: Red Hat KVM, BIOS 1.16.0-3.module_el8.7.0+1218+f626c2ff 04/01/2014
task: ffff8802c09a9280 ti: ffff8802af574000 task.ti: ffff8802af574000
RIP: 0010:[<ffffffffa13c5c33>] [<ffffffffa13c5c33>] lod_object_unlock+0x113/0x3c0 [lod]
RSP: 0018:ffff8802af577aa0 EFLAGS: 00010206
RAX: ffff8802ba0633c8 RBX: 0000000000000000 RCX: ffff8802b1806db0
RDX: 0000000000000000 RSI: 5a5a5a5a5a5a5a5a RDI: ffff8802c09fd458
RBP: ffff8802af577ac8 R08: 0000000000000001 R09: 000000000000ffff
R10: 0000000000000000 R11: ffff8802af5776a6 R12: ffff8802c09fd458
R13: ffff8802b18070e0 R14: ffff8802dcba73f8 R15: ffff8802b204c968
FS: 0000000000000000(0000) GS:ffff880331a00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007ff88dc04d70 CR3: 0000000001c10000 CR4: 00000000000007f0
Call Trace:
[<ffffffffa11f9b8d>] mdd_object_unlock+0x2d/0xd0 [mdd]
[<ffffffffa129dde2>] mdt_object_stripes_unlock+0x172/0x2d0 [mdt]
[<ffffffffa12a53ee>] mdt_reint_setattr+0xcbe/0x15f0 [mdt]
[<ffffffffa12a5da7>] mdt_reint_rec+0x87/0x240 [mdt]
[<ffffffffa1279b3c>] mdt_reint_internal+0x74c/0xbc0 [mdt]
[<ffffffffa1281cd5>] ? mdt_thread_info_init+0xa5/0xc0 [mdt]
[<ffffffffa12849f7>] mdt_reint+0x67/0x150 [mdt]
[<ffffffffa06f3d0e>] tgt_request_handle+0x74e/0x1a60 [ptlrpc]
[<ffffffffa0638ab1>] ptlrpc_server_handle_request+0x281/0xce0 [ptlrpc]
[<ffffffffa063a881>] ptlrpc_main+0xc61/0x1640 [ptlrpc]
[<ffffffff814119f9>] ? do_raw_spin_unlock+0x49/0x90
[<ffffffff817e329e>] ? _raw_spin_unlock_irq+0xe/0x30
[<ffffffffa0639c20>] ? ptlrpc_wait_event+0x630/0x630 [ptlrpc]
[<ffffffff810ba114>] kthread+0xe4/0xf0
[<ffffffff810ba030>] ? kthread_create_on_node+0x140/0x140
[<ffffffff817ede5d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff810ba030>] ? kthread_create_on_node+0x140/0x140

getfattr (28479) used greatest stack depth: 10144 bytes left
LustreError: 15137:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/14 failed: rc = -114
LustreError: 28454:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802cd828958: inode [0x200000403:0x9:0x0] mdc close failed: rc = -116
LustreError: 15137:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x2:0x0]/8 failed: rc = -114
LustreError: 27875:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802d2d18958: inode [0x200000404:0x2:0x0] mdc close failed: rc = -116
LustreError: 15134:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/16 failed: rc = -114
LustreError: 29718:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802cd828958: inode [0x240000403:0x5:0x0] mdc close failed: rc = -116
LustreError: 15131:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/1 failed: rc = -114
LustreError: 15131:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 2 previous similar messages
Lustre: 29130:0:(mdt_recovery.c:128:mdt_req_from_lrd()) @@@ restoring transno req@ffff8802b18f5040 x1823293866617472/t4294968040(0) o101->91397b0f-d7c1-4fe7-9615-1df305ca1bb5@0@lo:231/0 lens 376/816 e 0 to 0 dl 1738828651 ref 1 fl Interpret:H/202/0 rc 0/0 job:'dd.0' uid:0 gid:0
LustreError: 31499:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0001: migrate [0x240000403:0x1:0x0]/19 failed: rc = -2
LustreError: 31499:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 2 previous similar messages
LustreError: 31695:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x280000403:0x1:0x0]/5 failed: rc = -16
LustreError: 31695:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 3 previous similar messages
LustreError: 31499:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '10' migration was interrupted, run 'lfs migrate -m 1 -c 3 -H crush 10' to finish migration: rc = -1
Lustre: dir [0x280000403:0xd:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
LustreError: 31297:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802cd828958: inode [0x200000403:0x4a:0x0] mdc close failed: rc = -116
LustreError: 31297:0:(file.c:247:ll_close_inode_openhandle()) Skipped 1 previous similar message
LustreError: 2799:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x240000403:0xfb:0x0]: rc = -5
LustreError: 2799:0:(llite_lib.c:3735:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
Lustre: lustre-MDT0000: trigger partial OI scrub for RPC inconsistency, checking FID [0x200000404:0x72:0x0]/0xa): rc = 0
LustreError: 2854:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x240000403:0xfb:0x0]: rc = -5
LustreError: 2854:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 1 previous similar message
LustreError: 2854:0:(llite_lib.c:3735:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 2854:0:(llite_lib.c:3735:ll_prep_inode()) Skipped 1 previous similar message
Lustre: dir [0x200000404:0x45:0x0] stripe 1 readdir failed: -2, directory is partially accessed!
LustreError: 2650:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0002: migrate [0x280000404:0xd6:0x0]/5 failed: rc = -114
LustreError: 2650:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 19 previous similar messages
LustreError: 8164:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802cd828958: inode [0x280000403:0xcc:0x0] mdc close failed: rc = -2
LustreError: 8164:0:(file.c:247:ll_close_inode_openhandle()) Skipped 1 previous similar message
LustreError: 32082:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '16' migration was interrupted, run 'lfs migrate -m 0 -c 3 -H crush 16' to finish migration: rc = -1
Lustre: dir [0x240000403:0x50:0x0] stripe 2 readdir failed: -2, directory is partially accessed!
Lustre: dir [0x200000404:0x72:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 2 previous similar messages
LustreError: 6247:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802cd828958: inode [0x200000404:0x176:0x0] mdc close failed: rc = -116
LustreError: 18946:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x280000403:0xf1:0x0]: rc = -5
LustreError: 18946:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 3 previous similar messages
LustreError: 18946:0:(llite_lib.c:3735:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 18946:0:(llite_lib.c:3735:ll_prep_inode()) Skipped 3 previous similar messages
LustreError: 32082:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '16' migration was interrupted, run 'lfs migrate -m 0 -c 3 -H crush 16' to finish migration: rc = -1
Lustre: 29351:0:(mdt_recovery.c:128:mdt_req_from_lrd()) @@@ restoring transno req@ffff880095b71e40 x1823293874959744/t4294971565(0) o101->91397b0f-d7c1-4fe7-9615-1df305ca1bb5@0@lo:291/0 lens 376/864 e 0 to 0 dl 1738828711 ref 1 fl Interpret:H/202/0 rc 0/0 job:'dd.0' uid:0 gid:0
LustreError: 19892:0:(lcommon_cl.c:177:cl_file_inode_init()) lustre: failed to initialize cl_object [0x200000404:0x1fd:0x0]: rc = -5
LustreError: 19892:0:(lcommon_cl.c:177:cl_file_inode_init()) Skipped 3 previous similar messages
LustreError: 19892:0:(llite_lib.c:3735:ll_prep_inode()) lustre: new_inode - fatal error: rc = -5
LustreError: 19892:0:(llite_lib.c:3735:ll_prep_inode()) Skipped 3 previous similar messages
LustreError: 31695:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x240000404:0xf1:0x0]/9 failed: rc = -114
LustreError: 31695:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 25 previous similar messages
15[20308]: segfault at 8 ip 00007fd9bdb8a7e8 sp 00007ffd1c140a40 error 4 in ld-2.17.so[7fd9bdb7f000+22000]
LustreError: 15134:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0000: '16' migration was interrupted, run 'lfs migrate -m 0 -c 3 -H crush 16' to finish migration: rc = -1
LustreError: 6462:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802d2d18958: inode [0x240000404:0x9e:0x0] mdc close failed: rc = -2
LustreError: 6462:0:(file.c:247:ll_close_inode_openhandle()) Skipped 3 previous similar messages
Lustre: dir [0x200000403:0x210:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 4 previous similar messages
LustreError: 2650:0:(mdd_dir.c:4619:mdd_migrate_cmd_check()) lustre-MDD0002: '16' migration was interrupted, run 'lfs migrate -m 0 -c 3 -H crush 16' to finish migration: rc = -1
Lustre: dir [0x240000403:0x1c3:0x0] stripe 3 readdir failed: -2, directory is partially accessed!
Lustre: Skipped 3 previous similar messages
LustreError: 26689:0:(file.c:247:ll_close_inode_openhandle()) lustre-clilmv-ffff8802d2d18958: inode [0x240000403:0x27e:0x0] mdc close failed: rc = -116
LustreError: 15113:0:(mdt_xattr.c:402:mdt_dir_layout_update()) lustre-MDT0000: [0x200000404:0x18c:0x0] migrate mdt count mismatch 2 != 3
LustreError: 2650:0:(mdt_reint.c:2513:mdt_reint_migrate()) lustre-MDT0000: migrate [0x200000403:0x1:0x0]/10 failed: rc = -114
LustreError: 2650:0:(mdt_reint.c:2513:mdt_reint_migrate()) Skipped 51 previous similar messages