searchusermenu
  • 发布文章
  • 消息中心
点赞
收藏
评论
分享
原创

内核crash-efi固件运行时故障排查

2023-11-20 05:45:47
150
0

问题描述:

  1. ctyunos系统,鲲鹏机器
  2. 执行reboot时,100%必现crash

堆栈如下:

[ 1396.877810] reboot: Restarting system

[ 1396.881481] Unable to handle kernel paging request at virtual address 000000003ee0c440
[ 1396.889384] Mem abort info:
[ 1396.892164] ESR = 0x86000007
[ 1396.895205] Exception class = IABT (current EL), IL = 32 bits
[ 1396.901110] SET = 0, FnV = 0
[ 1396.904150] EA = 0, S1PTW = 0
[ 1396.907278] user pgtable: 64k pages, 48-bit VAs, pgdp = 0000000079f344ec
[ 1396.913965] [000000003ee0c440] pgd=0000000000000000, pud=0000000000000000
[ 1396.920740] Internal error: Oops: 86000007 [#1] SMP
[ 1396.925604] Modules linked in: xt_CHECKSUM ipt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 ip6table_mangle ip6table_nat nf_nat_ipv6 iptable_mangle iptable_nat nf_nat_ipv4 nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bridge stp llc tun cuse fuse scsi_transport_iscsi rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) tls strparser mlxdevm(OE) auxiliary(OE) ib_uverbs(OE) ib_core(OE) mlx_compat(OE) psample devlink rfkill sunrpc vfat fat ipmi_ssif ofpart aes_ce_blk cmdlinepart crypto_simd cryptd ses aes_ce_cipher hi_sfc mtd enclosure ghash_ce ipmi_si spi_dw_mmio sg ipmi_devintf sha2_ce sbsa_gwdt sha256_arm64 sha1_ce ipmi_msghandler sch_fq_codel ip_tables ext4 mbcache
[ 1396.996563] jbd2 sd_mod hisi_sas_v3_hw hisi_sas_main hclge libsas ahci scsi_transport_sas libahci hns3 megaraid_sas libata hnae3 nfit i2c_designware_platform i2c_designware_core libnvdimm dm_mirror dm_region_hash dm_log dm_mod [last unloaded: mlxfw]
[ 1397.018710] Process reboot (pid: 3931, stack limit = 0x00000000ff625b1d)
[ 1397.025399] CPU: 0 PID: 3931 Comm: reboot Kdump: loaded Tainted: G OE 4.19.90-2102.2.0.0066.ctl2.aarch64 #1
[ 1397.036339] Hardware name: To be filled by O.E.M. S920S10/BC82AMDT, BIOS 09.04.01.01.02 08/15/2023
[ 1397.045283] pstate: 80400089 (Nzcv daIf +PAN -UAO)
[ 1397.050062] pc : 0x3ee0c440
[ 1397.052842] lr : 0x207512bc
[ 1397.055621] sp : ffff000024e6fb10
[ 1397.058922] x29: ffff000024e6fb10 x28: ffffdfa4d84f8380
[ 1397.064220] x27: 0000000000000000 x26: 0000000000000000
[ 1397.069517] x25: 00000000207bfa98 x24: 00000000204a0028
[ 1397.074815] x23: 0000000000000000 x22: 0000000000000000
[ 1397.080112] x21: 0000000000000000 x20: 0000000000000000
[ 1397.085410] x19: 00000000207bded0 x18: ffffffffffffffff
[ 1397.090708] x17: 0000000000000000 x16: 00000000207511b8
[ 1397.096006] x15: ffff4c5b2e993708 x14: 0000000000000000
[ 1397.101303] x13: 0000000000000000 x12: 0000000000000000
[ 1397.106601] x11: 0000000000000000 x10: 0000000000000000
[ 1397.111898] x9 : 0000000000000000 x8 : 0000000020490a78
[ 1397.117196] x7 : 0000000000000000 x6 : 0000000000000000
[ 1397.122493] x5 : 000000003ee0c440 x4 : 0000000000000000
[ 1397.127791] x3 : 00000000204a0028 x2 : 0000000000000000
[ 1397.133088] x1 : 000000000311100a x0 : 0000000000000001
[ 1397.138386] Call trace:
[ 1397.140819] 0x3ee0c440
[ 1397.143251] 0x20490ad0
[ 1397.145687] __efi_rt_asm_wrapper+0x28/0x40
[ 1397.149857] virt_efi_reset_system+0x80/0xc8
[ 1397.154113] efi_reboot+0xb4/0x120
[ 1397.157502] machine_restart+0x7c/0x80
[ 1397.161238] kernel_restart+0x74/0x80
[ 1397.164886] __se_sys_reboot+0x214/0x260
[ 1397.168795] __arm64_sys_reboot+0x24/0x30
[ 1397.172792] el0_svc_common+0x78/0x138
[ 1397.176528] el0_svc_handler+0x38/0x78
[ 1397.180263] el0_svc+0x8/0x1f8
[ 1397.183305] Code: bad PC value
[ 1397.186893] Starting crashdump kernel...
[ 1397.190805] Bye!


EFI runtime service是一种由固件提供的功能,可以在任何CPU模式和任何时间使用,即使在退出UEFI boot service之后也可以。EFI runtime service可以为操作系统提供一些有用的功能,例如重启/关闭系统,获取/设置当前时间,管理变量等。
34 void efi_reboot(enum reboot_mode reboot_mode, const char *__unused)
33 {
32 |---const char *str[] = { "cold", "warm", "shutdown", "platform" };
31 |---int efi_mode, cap_reset_mode;
30
29 |---if (!efi_enabled(EFI_RUNTIME_SERVICES))
28 |---|---return;
27
26 |---switch (reboot_mode) {
25 |---case REBOOT_WARM:
24 |---case REBOOT_SOFT:
23 |---|---efi_mode = EFI_RESET_WARM;
22 |---|---break;
21 |---default:
20 |---|---efi_mode = EFI_RESET_COLD;
19 |---|---break;
18 |---}
17
16 |---/*
15 |--- * If a quirk forced an EFI reset mode, always use that.
14 |--- */
13 |---if (efi_reboot_quirk_mode != -1)
12 |---|---efi_mode = efi_reboot_quirk_mode;
11
10 |---if (efi_capsule_pending(&cap_reset_mode)) {
9 |---|---if (efi_mode != cap_reset_mode)
8 |---|---|---printk(KERN_CRIT "efi: %s reset requested but pending "
7 |---|---|--- "capsule update requires %s reset... Performing "
6 |---|---|--- "%s reset.\n", str[efi_mode], str[cap_reset_mode],
5 |---|---|--- str[cap_reset_mode]);
4 |---|---efi_mode = cap_reset_mode;
3 |---}
2
1 |---efi.reset_system(efi_mode, EFI_SUCCESS, 0, NULL);
47 }


14 static void virt_efi_reset_system(int reset_type,
13 |---|---|---|--- efi_status_t status,
12 |---|---|---|--- unsigned long data_size,
11 |---|---|---|--- efi_char16_t *data)
10 {
9 |---if (down_interruptible(&efi_runtime_lock)) {
8 |---|---pr_warn("failed to invoke the reset_system() runtime service:\n"
7 |---|---|---"could not get exclusive access to the firmware\n");
6 |---|---return;
5 |---}
4 |---__efi_call_virt(reset_system, reset_type, status, data_size, data);
3 |---up(&efi_runtime_lock);
2 }


45 #define __efi_call_virt(f, args...) \
1 |---__efi_call_virt_pointer(efi.systab->runtime, f, args)

20 #define __efi_call_virt_pointer(p, f, args...)|-|---|---|---\
19 ({|-|---|---|---|---|---|---|---|---\
18 |---unsigned long __flags;|-|---|---|---|---|---\
17 |---|---|---|---|---|---|---|---|---\
16 |---arch_efi_call_virt_setup();||---|---|---|---\
15 |---|---|---|---|---|---|---|---|---\
14 |---__flags = efi_call_virt_save_flags();|--|---|---|---\
13 |---arch_efi_call_virt(p, f, args);||---|---|---|---\
12 |---efi_call_virt_check_flags(__flags, __stringify(f));||---\
11 |---|---|---|---|---|---|---|---|---\
10 |---arch_efi_call_virt_teardown();|-|---|---|---|---\
9 })


10 #define arch_efi_call_virt(p, f, args...)|--|---|---|---\
9 ({|-|---|---|---|---|---|---|---|---\
8 |---efi_##f##_t *__f;|--|---|---|---|---|---\
7 |---__f = p->f;||---|---|---|---|---|---\
6 |---__efi_rt_asm_wrapper(__f, #f, args);|---|---|---|---\
5 }

efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...);


ENTRY(__efi_rt_asm_wrapper)
stp x29, x30, [sp, #-32]!
mov x29, sp

/*
* Register x18 is designated as the 'platform' register by the AAPCS,
* which means firmware running at the same exception level as the OS
* (such as UEFI) should never touch it.
*/
stp x1, x18, [sp, #16]

/*
* We are lucky enough that no EFI runtime services take more than
* 5 arguments, so all are passed in registers rather than via the
* stack.
*/
mov x8, x0
mov x0, x2
mov x1, x3
mov x2, x4
mov x3, x5
mov x4, x6
blr x8

ldp x1, x2, [sp, #16]
cmp x2, x18
ldp x29, x30, [sp], #32
b.ne 0f
ret
0: b efi_handle_corrupted_x18 // tail call
ENDPROC(__efi_rt_asm_wrapper)

 

 

验证测试,

1.常用centos7/8 、ubuntu 18.04-22.10 都会出现crash

2. openeuler全系列crash

3. debian 最新版,6.10内核正常

 

结论:

 

1. 6.* 内核增加了对固件运行时的异常处理,如果固件存在缺陷,会调用其他方式处理

2. 其他系统没有考虑到固件运行错误,均会crash

3. efi=noruntime配置后不会crash

4. 根因:固件问题,升级固件后问题也解决。若无法规避则修改内核参数

0条评论
0 / 1000
y****n
11文章数
0粉丝数
y****n
11 文章 | 0 粉丝
原创

内核crash-efi固件运行时故障排查

2023-11-20 05:45:47
150
0

问题描述:

  1. ctyunos系统,鲲鹏机器
  2. 执行reboot时,100%必现crash

堆栈如下:

[ 1396.877810] reboot: Restarting system

[ 1396.881481] Unable to handle kernel paging request at virtual address 000000003ee0c440
[ 1396.889384] Mem abort info:
[ 1396.892164] ESR = 0x86000007
[ 1396.895205] Exception class = IABT (current EL), IL = 32 bits
[ 1396.901110] SET = 0, FnV = 0
[ 1396.904150] EA = 0, S1PTW = 0
[ 1396.907278] user pgtable: 64k pages, 48-bit VAs, pgdp = 0000000079f344ec
[ 1396.913965] [000000003ee0c440] pgd=0000000000000000, pud=0000000000000000
[ 1396.920740] Internal error: Oops: 86000007 [#1] SMP
[ 1396.925604] Modules linked in: xt_CHECKSUM ipt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 ip6table_mangle ip6table_nat nf_nat_ipv6 iptable_mangle iptable_nat nf_nat_ipv4 nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 libcrc32c ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bridge stp llc tun cuse fuse scsi_transport_iscsi rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) tls strparser mlxdevm(OE) auxiliary(OE) ib_uverbs(OE) ib_core(OE) mlx_compat(OE) psample devlink rfkill sunrpc vfat fat ipmi_ssif ofpart aes_ce_blk cmdlinepart crypto_simd cryptd ses aes_ce_cipher hi_sfc mtd enclosure ghash_ce ipmi_si spi_dw_mmio sg ipmi_devintf sha2_ce sbsa_gwdt sha256_arm64 sha1_ce ipmi_msghandler sch_fq_codel ip_tables ext4 mbcache
[ 1396.996563] jbd2 sd_mod hisi_sas_v3_hw hisi_sas_main hclge libsas ahci scsi_transport_sas libahci hns3 megaraid_sas libata hnae3 nfit i2c_designware_platform i2c_designware_core libnvdimm dm_mirror dm_region_hash dm_log dm_mod [last unloaded: mlxfw]
[ 1397.018710] Process reboot (pid: 3931, stack limit = 0x00000000ff625b1d)
[ 1397.025399] CPU: 0 PID: 3931 Comm: reboot Kdump: loaded Tainted: G OE 4.19.90-2102.2.0.0066.ctl2.aarch64 #1
[ 1397.036339] Hardware name: To be filled by O.E.M. S920S10/BC82AMDT, BIOS 09.04.01.01.02 08/15/2023
[ 1397.045283] pstate: 80400089 (Nzcv daIf +PAN -UAO)
[ 1397.050062] pc : 0x3ee0c440
[ 1397.052842] lr : 0x207512bc
[ 1397.055621] sp : ffff000024e6fb10
[ 1397.058922] x29: ffff000024e6fb10 x28: ffffdfa4d84f8380
[ 1397.064220] x27: 0000000000000000 x26: 0000000000000000
[ 1397.069517] x25: 00000000207bfa98 x24: 00000000204a0028
[ 1397.074815] x23: 0000000000000000 x22: 0000000000000000
[ 1397.080112] x21: 0000000000000000 x20: 0000000000000000
[ 1397.085410] x19: 00000000207bded0 x18: ffffffffffffffff
[ 1397.090708] x17: 0000000000000000 x16: 00000000207511b8
[ 1397.096006] x15: ffff4c5b2e993708 x14: 0000000000000000
[ 1397.101303] x13: 0000000000000000 x12: 0000000000000000
[ 1397.106601] x11: 0000000000000000 x10: 0000000000000000
[ 1397.111898] x9 : 0000000000000000 x8 : 0000000020490a78
[ 1397.117196] x7 : 0000000000000000 x6 : 0000000000000000
[ 1397.122493] x5 : 000000003ee0c440 x4 : 0000000000000000
[ 1397.127791] x3 : 00000000204a0028 x2 : 0000000000000000
[ 1397.133088] x1 : 000000000311100a x0 : 0000000000000001
[ 1397.138386] Call trace:
[ 1397.140819] 0x3ee0c440
[ 1397.143251] 0x20490ad0
[ 1397.145687] __efi_rt_asm_wrapper+0x28/0x40
[ 1397.149857] virt_efi_reset_system+0x80/0xc8
[ 1397.154113] efi_reboot+0xb4/0x120
[ 1397.157502] machine_restart+0x7c/0x80
[ 1397.161238] kernel_restart+0x74/0x80
[ 1397.164886] __se_sys_reboot+0x214/0x260
[ 1397.168795] __arm64_sys_reboot+0x24/0x30
[ 1397.172792] el0_svc_common+0x78/0x138
[ 1397.176528] el0_svc_handler+0x38/0x78
[ 1397.180263] el0_svc+0x8/0x1f8
[ 1397.183305] Code: bad PC value
[ 1397.186893] Starting crashdump kernel...
[ 1397.190805] Bye!


EFI runtime service是一种由固件提供的功能,可以在任何CPU模式和任何时间使用,即使在退出UEFI boot service之后也可以。EFI runtime service可以为操作系统提供一些有用的功能,例如重启/关闭系统,获取/设置当前时间,管理变量等。
34 void efi_reboot(enum reboot_mode reboot_mode, const char *__unused)
33 {
32 |---const char *str[] = { "cold", "warm", "shutdown", "platform" };
31 |---int efi_mode, cap_reset_mode;
30
29 |---if (!efi_enabled(EFI_RUNTIME_SERVICES))
28 |---|---return;
27
26 |---switch (reboot_mode) {
25 |---case REBOOT_WARM:
24 |---case REBOOT_SOFT:
23 |---|---efi_mode = EFI_RESET_WARM;
22 |---|---break;
21 |---default:
20 |---|---efi_mode = EFI_RESET_COLD;
19 |---|---break;
18 |---}
17
16 |---/*
15 |--- * If a quirk forced an EFI reset mode, always use that.
14 |--- */
13 |---if (efi_reboot_quirk_mode != -1)
12 |---|---efi_mode = efi_reboot_quirk_mode;
11
10 |---if (efi_capsule_pending(&cap_reset_mode)) {
9 |---|---if (efi_mode != cap_reset_mode)
8 |---|---|---printk(KERN_CRIT "efi: %s reset requested but pending "
7 |---|---|--- "capsule update requires %s reset... Performing "
6 |---|---|--- "%s reset.\n", str[efi_mode], str[cap_reset_mode],
5 |---|---|--- str[cap_reset_mode]);
4 |---|---efi_mode = cap_reset_mode;
3 |---}
2
1 |---efi.reset_system(efi_mode, EFI_SUCCESS, 0, NULL);
47 }


14 static void virt_efi_reset_system(int reset_type,
13 |---|---|---|--- efi_status_t status,
12 |---|---|---|--- unsigned long data_size,
11 |---|---|---|--- efi_char16_t *data)
10 {
9 |---if (down_interruptible(&efi_runtime_lock)) {
8 |---|---pr_warn("failed to invoke the reset_system() runtime service:\n"
7 |---|---|---"could not get exclusive access to the firmware\n");
6 |---|---return;
5 |---}
4 |---__efi_call_virt(reset_system, reset_type, status, data_size, data);
3 |---up(&efi_runtime_lock);
2 }


45 #define __efi_call_virt(f, args...) \
1 |---__efi_call_virt_pointer(efi.systab->runtime, f, args)

20 #define __efi_call_virt_pointer(p, f, args...)|-|---|---|---\
19 ({|-|---|---|---|---|---|---|---|---\
18 |---unsigned long __flags;|-|---|---|---|---|---\
17 |---|---|---|---|---|---|---|---|---\
16 |---arch_efi_call_virt_setup();||---|---|---|---\
15 |---|---|---|---|---|---|---|---|---\
14 |---__flags = efi_call_virt_save_flags();|--|---|---|---\
13 |---arch_efi_call_virt(p, f, args);||---|---|---|---\
12 |---efi_call_virt_check_flags(__flags, __stringify(f));||---\
11 |---|---|---|---|---|---|---|---|---\
10 |---arch_efi_call_virt_teardown();|-|---|---|---|---\
9 })


10 #define arch_efi_call_virt(p, f, args...)|--|---|---|---\
9 ({|-|---|---|---|---|---|---|---|---\
8 |---efi_##f##_t *__f;|--|---|---|---|---|---\
7 |---__f = p->f;||---|---|---|---|---|---\
6 |---__efi_rt_asm_wrapper(__f, #f, args);|---|---|---|---\
5 }

efi_status_t __efi_rt_asm_wrapper(void *, const char *, ...);


ENTRY(__efi_rt_asm_wrapper)
stp x29, x30, [sp, #-32]!
mov x29, sp

/*
* Register x18 is designated as the 'platform' register by the AAPCS,
* which means firmware running at the same exception level as the OS
* (such as UEFI) should never touch it.
*/
stp x1, x18, [sp, #16]

/*
* We are lucky enough that no EFI runtime services take more than
* 5 arguments, so all are passed in registers rather than via the
* stack.
*/
mov x8, x0
mov x0, x2
mov x1, x3
mov x2, x4
mov x3, x5
mov x4, x6
blr x8

ldp x1, x2, [sp, #16]
cmp x2, x18
ldp x29, x30, [sp], #32
b.ne 0f
ret
0: b efi_handle_corrupted_x18 // tail call
ENDPROC(__efi_rt_asm_wrapper)

 

 

验证测试,

1.常用centos7/8 、ubuntu 18.04-22.10 都会出现crash

2. openeuler全系列crash

3. debian 最新版,6.10内核正常

 

结论:

 

1. 6.* 内核增加了对固件运行时的异常处理,如果固件存在缺陷,会调用其他方式处理

2. 其他系统没有考虑到固件运行错误,均会crash

3. efi=noruntime配置后不会crash

4. 根因:固件问题,升级固件后问题也解决。若无法规避则修改内核参数

文章来自个人专栏
文章 | 订阅
0条评论
0 / 1000
请输入你的评论
0
0