2019-05-19 15:51:43 +02:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2017-07-11 10:33:42 -05:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <string.h>
|
|
|
|
|
2020-09-04 16:30:27 +01:00
|
|
|
#include <linux/objtool.h>
|
|
|
|
#include <asm/orc_types.h>
|
|
|
|
|
2017-07-11 10:33:42 -05:00
|
|
|
#include "check.h"
|
|
|
|
#include "warn.h"
|
|
|
|
|
objtool: Introduce CFI hash
commit 8b946cc38e063f0f7bb67789478c38f6d7d457c9 upstream.
Andi reported that objtool on vmlinux.o consumes more memory than his
system has, leading to horrific performance.
This is in part because we keep a struct instruction for every
instruction in the file in-memory. Shrink struct instruction by
removing the CFI state (which includes full register state) from it
and demand allocating it.
Given most instructions don't actually change CFI state, there's lots
of repetition there, so add a hash table to find previous CFI
instances.
Reduces memory consumption (and runtime) for processing an
x86_64-allyesconfig:
pre: 4:40.84 real, 143.99 user, 44.18 sys, 30624988 mem
post: 2:14.61 real, 108.58 user, 25.04 sys, 16396184 mem
Suggested-by: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210624095147.756759107@infradead.org
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
[bwh: Backported to 5.10:
- Don't use bswap_if_needed() since we don't have any of the other fixes
for mixed-endian cross-compilation
- Since we don't have "objtool: Rewrite hashtable sizing", make
cfi_hash_alloc() set the number of bits similarly to elf_hash_bits()
- objtool doesn't have any mcount handling
- Adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-06-24 11:41:01 +02:00
|
|
|
static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi,
|
|
|
|
struct instruction *insn)
|
2017-07-11 10:33:42 -05:00
|
|
|
{
|
2020-12-17 15:02:42 -06:00
|
|
|
struct cfi_reg *bp = &cfi->regs[CFI_BP];
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
memset(orc, 0, sizeof(*orc));
|
2017-07-11 10:33:42 -05:00
|
|
|
|
objtool: Introduce CFI hash
commit 8b946cc38e063f0f7bb67789478c38f6d7d457c9 upstream.
Andi reported that objtool on vmlinux.o consumes more memory than his
system has, leading to horrific performance.
This is in part because we keep a struct instruction for every
instruction in the file in-memory. Shrink struct instruction by
removing the CFI state (which includes full register state) from it
and demand allocating it.
Given most instructions don't actually change CFI state, there's lots
of repetition there, so add a hash table to find previous CFI
instances.
Reduces memory consumption (and runtime) for processing an
x86_64-allyesconfig:
pre: 4:40.84 real, 143.99 user, 44.18 sys, 30624988 mem
post: 2:14.61 real, 108.58 user, 25.04 sys, 16396184 mem
Suggested-by: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210624095147.756759107@infradead.org
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
[bwh: Backported to 5.10:
- Don't use bswap_if_needed() since we don't have any of the other fixes
for mixed-endian cross-compilation
- Since we don't have "objtool: Rewrite hashtable sizing", make
cfi_hash_alloc() set the number of bits similarly to elf_hash_bits()
- objtool doesn't have any mcount handling
- Adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-06-24 11:41:01 +02:00
|
|
|
if (!cfi) {
|
|
|
|
orc->end = 0;
|
|
|
|
orc->sp_reg = ORC_REG_UNDEFINED;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
orc->end = cfi->end;
|
2018-05-18 08:47:12 +02:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
if (cfi->cfa.base == CFI_UNDEFINED) {
|
|
|
|
orc->sp_reg = ORC_REG_UNDEFINED;
|
|
|
|
return 0;
|
|
|
|
}
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
switch (cfi->cfa.base) {
|
|
|
|
case CFI_SP:
|
|
|
|
orc->sp_reg = ORC_REG_SP;
|
|
|
|
break;
|
|
|
|
case CFI_SP_INDIRECT:
|
|
|
|
orc->sp_reg = ORC_REG_SP_INDIRECT;
|
|
|
|
break;
|
|
|
|
case CFI_BP:
|
|
|
|
orc->sp_reg = ORC_REG_BP;
|
|
|
|
break;
|
|
|
|
case CFI_BP_INDIRECT:
|
|
|
|
orc->sp_reg = ORC_REG_BP_INDIRECT;
|
|
|
|
break;
|
|
|
|
case CFI_R10:
|
|
|
|
orc->sp_reg = ORC_REG_R10;
|
|
|
|
break;
|
|
|
|
case CFI_R13:
|
|
|
|
orc->sp_reg = ORC_REG_R13;
|
|
|
|
break;
|
|
|
|
case CFI_DI:
|
|
|
|
orc->sp_reg = ORC_REG_DI;
|
|
|
|
break;
|
|
|
|
case CFI_DX:
|
|
|
|
orc->sp_reg = ORC_REG_DX;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
WARN_FUNC("unknown CFA base reg %d",
|
|
|
|
insn->sec, insn->offset, cfi->cfa.base);
|
|
|
|
return -1;
|
|
|
|
}
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
switch (bp->base) {
|
|
|
|
case CFI_UNDEFINED:
|
|
|
|
orc->bp_reg = ORC_REG_UNDEFINED;
|
|
|
|
break;
|
|
|
|
case CFI_CFA:
|
|
|
|
orc->bp_reg = ORC_REG_PREV_SP;
|
|
|
|
break;
|
|
|
|
case CFI_BP:
|
|
|
|
orc->bp_reg = ORC_REG_BP;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
WARN_FUNC("unknown BP base reg %d",
|
|
|
|
insn->sec, insn->offset, bp->base);
|
|
|
|
return -1;
|
2017-07-11 10:33:42 -05:00
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
orc->sp_offset = cfi->cfa.offset;
|
|
|
|
orc->bp_offset = bp->offset;
|
|
|
|
orc->type = cfi->type;
|
|
|
|
|
2017-07-11 10:33:42 -05:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
static int write_orc_entry(struct elf *elf, struct section *orc_sec,
|
2021-03-26 16:12:07 +01:00
|
|
|
struct section *ip_sec, unsigned int idx,
|
2020-12-17 15:02:42 -06:00
|
|
|
struct section *insn_sec, unsigned long insn_off,
|
|
|
|
struct orc_entry *o)
|
2017-07-11 10:33:42 -05:00
|
|
|
{
|
|
|
|
struct orc_entry *orc;
|
|
|
|
|
|
|
|
/* populate ORC data */
|
2020-12-17 15:02:42 -06:00
|
|
|
orc = (struct orc_entry *)orc_sec->data->d_buf + idx;
|
2017-07-11 10:33:42 -05:00
|
|
|
memcpy(orc, o, sizeof(*orc));
|
|
|
|
|
objtool: Rename rela to reloc
Before supporting additional relocation types rename the relevant
types and functions from "rela" to "reloc". This work be done with
the following regex:
sed -e 's/struct rela/struct reloc/g' \
-e 's/\([_\*]\)rela\(s\{0,1\}\)/\1reloc\2/g' \
-e 's/tmprela\(s\{0,1\}\)/tmpreloc\1/g' \
-e 's/relasec/relocsec/g' \
-e 's/rela_list/reloc_list/g' \
-e 's/rela_hash/reloc_hash/g' \
-e 's/add_rela/add_reloc/g' \
-e 's/rela->/reloc->/g' \
-e '/rela[,\.]/{ s/\([^\.>]\)rela\([\.,]\)/\1reloc\2/g ; }' \
-e 's/rela =/reloc =/g' \
-e 's/relas =/relocs =/g' \
-e 's/relas\[/relocs[/g' \
-e 's/relaname =/relocname =/g' \
-e 's/= rela\;/= reloc\;/g' \
-e 's/= relas\;/= relocs\;/g' \
-e 's/= relaname\;/= relocname\;/g' \
-e 's/, rela)/, reloc)/g' \
-e 's/\([ @]\)rela\([ "]\)/\1reloc\2/g' \
-e 's/ rela$/ reloc/g' \
-e 's/, relaname/, relocname/g' \
-e 's/sec->rela/sec->reloc/g' \
-e 's/(\(!\{0,1\}\)rela/(\1reloc/g' \
-i \
arch.h \
arch/x86/decode.c \
check.c \
check.h \
elf.c \
elf.h \
orc_gen.c \
special.c
Notable exceptions which complicate the regex include gelf_*
library calls and standard/expected section names which still use
"rela" because they encode the type of relocation expected. Also, keep
"rela" in the struct because it encodes a specific type of relocation
we currently expect.
It will eventually turn into a member of an anonymous union when a
susequent patch adds implicit addend, or "rel", relocation support.
Signed-off-by: Matt Helsley <mhelsley@vmware.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
2020-05-29 14:01:13 -07:00
|
|
|
/* populate reloc for ip */
|
2021-03-26 16:12:07 +01:00
|
|
|
if (elf_add_reloc_to_insn(elf, ip_sec, idx * sizeof(int), R_X86_64_PC32,
|
|
|
|
insn_sec, insn_off))
|
2017-07-11 10:33:42 -05:00
|
|
|
return -1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
struct orc_list_entry {
|
|
|
|
struct list_head list;
|
|
|
|
struct orc_entry orc;
|
|
|
|
struct section *insn_sec;
|
|
|
|
unsigned long insn_off;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int orc_list_add(struct list_head *orc_list, struct orc_entry *orc,
|
|
|
|
struct section *sec, unsigned long offset)
|
|
|
|
{
|
|
|
|
struct orc_list_entry *entry = malloc(sizeof(*entry));
|
|
|
|
|
|
|
|
if (!entry) {
|
|
|
|
WARN("malloc failed");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
entry->orc = *orc;
|
|
|
|
entry->insn_sec = sec;
|
|
|
|
entry->insn_off = offset;
|
|
|
|
|
|
|
|
list_add_tail(&entry->list, orc_list);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
objtool: Support stack layout changes in alternatives
commit c9c324dc22aab1687da37001b321b6dfa93a0699 upstream.
The ORC unwinder showed a warning [1] which revealed the stack layout
didn't match what was expected. The problem was that paravirt patching
had replaced "CALL *pv_ops.irq.save_fl" with "PUSHF;POP". That changed
the stack layout between the PUSHF and the POP, so unwinding from an
interrupt which occurred between those two instructions would fail.
Part of the agreed upon solution was to rework the custom paravirt
patching code to use alternatives instead, since objtool already knows
how to read alternatives (and converging runtime patching infrastructure
is always a good thing anyway). But the main problem still remains,
which is that runtime patching can change the stack layout.
Making stack layout changes in alternatives was disallowed with commit
7117f16bf460 ("objtool: Fix ORC vs alternatives"), but now that paravirt
is going to be doing it, it needs to be supported.
One way to do so would be to modify the ORC table when the code gets
patched. But ORC is simple -- a good thing! -- and it's best to leave
it alone.
Instead, support stack layout changes by "flattening" all possible stack
states (CFI) from parallel alternative code streams into a single set of
linear states. The only necessary limitation is that CFI conflicts are
disallowed at all possible instruction boundaries.
For example, this scenario is allowed:
Alt1 Alt2 Alt3
0x00 CALL *pv_ops.save_fl CALL xen_save_fl PUSHF
0x01 POP %RAX
0x02 NOP
...
0x05 NOP
...
0x07 <insn>
The unwind information for offset-0x00 is identical for all 3
alternatives. Similarly offset-0x05 and higher also are identical (and
the same as 0x00). However offset-0x01 has deviating CFI, but that is
only relevant for Alt3, neither of the other alternative instruction
streams will ever hit that offset.
This scenario is NOT allowed:
Alt1 Alt2
0x00 CALL *pv_ops.save_fl PUSHF
0x01 NOP6
...
0x07 NOP POP %RAX
The problem here is that offset-0x7, which is an instruction boundary in
both possible instruction patch streams, has two conflicting stack
layouts.
[ The above examples were stolen from Peter Zijlstra. ]
The new flattened CFI array is used both for the detection of conflicts
(like the second example above) and the generation of linear ORC
entries.
BTW, another benefit of these changes is that, thanks to some related
cleanups (new fake nops and alt_group struct) objtool can finally be rid
of fake jumps, which were a constant source of headaches.
[1] https://lkml.kernel.org/r/20201111170536.arx2zbn4ngvjoov7@treble
Cc: Shinichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2020-12-18 14:26:21 -06:00
|
|
|
static unsigned long alt_group_len(struct alt_group *alt_group)
|
|
|
|
{
|
|
|
|
return alt_group->last_insn->offset +
|
|
|
|
alt_group->last_insn->len -
|
|
|
|
alt_group->first_insn->offset;
|
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
int orc_create(struct objtool_file *file)
|
2017-07-11 10:33:42 -05:00
|
|
|
{
|
2021-03-26 16:12:07 +01:00
|
|
|
struct section *sec, *orc_sec;
|
2020-12-17 15:02:42 -06:00
|
|
|
unsigned int nr = 0, idx = 0;
|
|
|
|
struct orc_list_entry *entry;
|
|
|
|
struct list_head orc_list;
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
struct orc_entry null = {
|
|
|
|
.sp_reg = ORC_REG_UNDEFINED,
|
2017-07-11 10:33:42 -05:00
|
|
|
.bp_reg = ORC_REG_UNDEFINED,
|
2020-09-04 16:30:27 +01:00
|
|
|
.type = UNWIND_HINT_TYPE_CALL,
|
2017-07-11 10:33:42 -05:00
|
|
|
};
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
/* Build a deduplicated list of ORC entries: */
|
|
|
|
INIT_LIST_HEAD(&orc_list);
|
2017-07-11 10:33:42 -05:00
|
|
|
for_each_sec(file, sec) {
|
2020-12-17 15:02:42 -06:00
|
|
|
struct orc_entry orc, prev_orc = {0};
|
|
|
|
struct instruction *insn;
|
|
|
|
bool empty = true;
|
|
|
|
|
2017-07-11 10:33:42 -05:00
|
|
|
if (!sec->text)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
sec_for_each_insn(file, sec, insn) {
|
objtool: Support stack layout changes in alternatives
commit c9c324dc22aab1687da37001b321b6dfa93a0699 upstream.
The ORC unwinder showed a warning [1] which revealed the stack layout
didn't match what was expected. The problem was that paravirt patching
had replaced "CALL *pv_ops.irq.save_fl" with "PUSHF;POP". That changed
the stack layout between the PUSHF and the POP, so unwinding from an
interrupt which occurred between those two instructions would fail.
Part of the agreed upon solution was to rework the custom paravirt
patching code to use alternatives instead, since objtool already knows
how to read alternatives (and converging runtime patching infrastructure
is always a good thing anyway). But the main problem still remains,
which is that runtime patching can change the stack layout.
Making stack layout changes in alternatives was disallowed with commit
7117f16bf460 ("objtool: Fix ORC vs alternatives"), but now that paravirt
is going to be doing it, it needs to be supported.
One way to do so would be to modify the ORC table when the code gets
patched. But ORC is simple -- a good thing! -- and it's best to leave
it alone.
Instead, support stack layout changes by "flattening" all possible stack
states (CFI) from parallel alternative code streams into a single set of
linear states. The only necessary limitation is that CFI conflicts are
disallowed at all possible instruction boundaries.
For example, this scenario is allowed:
Alt1 Alt2 Alt3
0x00 CALL *pv_ops.save_fl CALL xen_save_fl PUSHF
0x01 POP %RAX
0x02 NOP
...
0x05 NOP
...
0x07 <insn>
The unwind information for offset-0x00 is identical for all 3
alternatives. Similarly offset-0x05 and higher also are identical (and
the same as 0x00). However offset-0x01 has deviating CFI, but that is
only relevant for Alt3, neither of the other alternative instruction
streams will ever hit that offset.
This scenario is NOT allowed:
Alt1 Alt2
0x00 CALL *pv_ops.save_fl PUSHF
0x01 NOP6
...
0x07 NOP POP %RAX
The problem here is that offset-0x7, which is an instruction boundary in
both possible instruction patch streams, has two conflicting stack
layouts.
[ The above examples were stolen from Peter Zijlstra. ]
The new flattened CFI array is used both for the detection of conflicts
(like the second example above) and the generation of linear ORC
entries.
BTW, another benefit of these changes is that, thanks to some related
cleanups (new fake nops and alt_group struct) objtool can finally be rid
of fake jumps, which were a constant source of headaches.
[1] https://lkml.kernel.org/r/20201111170536.arx2zbn4ngvjoov7@treble
Cc: Shinichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2020-12-18 14:26:21 -06:00
|
|
|
struct alt_group *alt_group = insn->alt_group;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!alt_group) {
|
objtool: Introduce CFI hash
commit 8b946cc38e063f0f7bb67789478c38f6d7d457c9 upstream.
Andi reported that objtool on vmlinux.o consumes more memory than his
system has, leading to horrific performance.
This is in part because we keep a struct instruction for every
instruction in the file in-memory. Shrink struct instruction by
removing the CFI state (which includes full register state) from it
and demand allocating it.
Given most instructions don't actually change CFI state, there's lots
of repetition there, so add a hash table to find previous CFI
instances.
Reduces memory consumption (and runtime) for processing an
x86_64-allyesconfig:
pre: 4:40.84 real, 143.99 user, 44.18 sys, 30624988 mem
post: 2:14.61 real, 108.58 user, 25.04 sys, 16396184 mem
Suggested-by: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210624095147.756759107@infradead.org
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
[bwh: Backported to 5.10:
- Don't use bswap_if_needed() since we don't have any of the other fixes
for mixed-endian cross-compilation
- Since we don't have "objtool: Rewrite hashtable sizing", make
cfi_hash_alloc() set the number of bits similarly to elf_hash_bits()
- objtool doesn't have any mcount handling
- Adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-06-24 11:41:01 +02:00
|
|
|
if (init_orc_entry(&orc, insn->cfi, insn))
|
objtool: Support stack layout changes in alternatives
commit c9c324dc22aab1687da37001b321b6dfa93a0699 upstream.
The ORC unwinder showed a warning [1] which revealed the stack layout
didn't match what was expected. The problem was that paravirt patching
had replaced "CALL *pv_ops.irq.save_fl" with "PUSHF;POP". That changed
the stack layout between the PUSHF and the POP, so unwinding from an
interrupt which occurred between those two instructions would fail.
Part of the agreed upon solution was to rework the custom paravirt
patching code to use alternatives instead, since objtool already knows
how to read alternatives (and converging runtime patching infrastructure
is always a good thing anyway). But the main problem still remains,
which is that runtime patching can change the stack layout.
Making stack layout changes in alternatives was disallowed with commit
7117f16bf460 ("objtool: Fix ORC vs alternatives"), but now that paravirt
is going to be doing it, it needs to be supported.
One way to do so would be to modify the ORC table when the code gets
patched. But ORC is simple -- a good thing! -- and it's best to leave
it alone.
Instead, support stack layout changes by "flattening" all possible stack
states (CFI) from parallel alternative code streams into a single set of
linear states. The only necessary limitation is that CFI conflicts are
disallowed at all possible instruction boundaries.
For example, this scenario is allowed:
Alt1 Alt2 Alt3
0x00 CALL *pv_ops.save_fl CALL xen_save_fl PUSHF
0x01 POP %RAX
0x02 NOP
...
0x05 NOP
...
0x07 <insn>
The unwind information for offset-0x00 is identical for all 3
alternatives. Similarly offset-0x05 and higher also are identical (and
the same as 0x00). However offset-0x01 has deviating CFI, but that is
only relevant for Alt3, neither of the other alternative instruction
streams will ever hit that offset.
This scenario is NOT allowed:
Alt1 Alt2
0x00 CALL *pv_ops.save_fl PUSHF
0x01 NOP6
...
0x07 NOP POP %RAX
The problem here is that offset-0x7, which is an instruction boundary in
both possible instruction patch streams, has two conflicting stack
layouts.
[ The above examples were stolen from Peter Zijlstra. ]
The new flattened CFI array is used both for the detection of conflicts
(like the second example above) and the generation of linear ORC
entries.
BTW, another benefit of these changes is that, thanks to some related
cleanups (new fake nops and alt_group struct) objtool can finally be rid
of fake jumps, which were a constant source of headaches.
[1] https://lkml.kernel.org/r/20201111170536.arx2zbn4ngvjoov7@treble
Cc: Shinichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2020-12-18 14:26:21 -06:00
|
|
|
return -1;
|
|
|
|
if (!memcmp(&prev_orc, &orc, sizeof(orc)))
|
|
|
|
continue;
|
|
|
|
if (orc_list_add(&orc_list, &orc, sec,
|
|
|
|
insn->offset))
|
|
|
|
return -1;
|
|
|
|
nr++;
|
|
|
|
prev_orc = orc;
|
|
|
|
empty = false;
|
2020-12-17 15:02:42 -06:00
|
|
|
continue;
|
objtool: Support stack layout changes in alternatives
commit c9c324dc22aab1687da37001b321b6dfa93a0699 upstream.
The ORC unwinder showed a warning [1] which revealed the stack layout
didn't match what was expected. The problem was that paravirt patching
had replaced "CALL *pv_ops.irq.save_fl" with "PUSHF;POP". That changed
the stack layout between the PUSHF and the POP, so unwinding from an
interrupt which occurred between those two instructions would fail.
Part of the agreed upon solution was to rework the custom paravirt
patching code to use alternatives instead, since objtool already knows
how to read alternatives (and converging runtime patching infrastructure
is always a good thing anyway). But the main problem still remains,
which is that runtime patching can change the stack layout.
Making stack layout changes in alternatives was disallowed with commit
7117f16bf460 ("objtool: Fix ORC vs alternatives"), but now that paravirt
is going to be doing it, it needs to be supported.
One way to do so would be to modify the ORC table when the code gets
patched. But ORC is simple -- a good thing! -- and it's best to leave
it alone.
Instead, support stack layout changes by "flattening" all possible stack
states (CFI) from parallel alternative code streams into a single set of
linear states. The only necessary limitation is that CFI conflicts are
disallowed at all possible instruction boundaries.
For example, this scenario is allowed:
Alt1 Alt2 Alt3
0x00 CALL *pv_ops.save_fl CALL xen_save_fl PUSHF
0x01 POP %RAX
0x02 NOP
...
0x05 NOP
...
0x07 <insn>
The unwind information for offset-0x00 is identical for all 3
alternatives. Similarly offset-0x05 and higher also are identical (and
the same as 0x00). However offset-0x01 has deviating CFI, but that is
only relevant for Alt3, neither of the other alternative instruction
streams will ever hit that offset.
This scenario is NOT allowed:
Alt1 Alt2
0x00 CALL *pv_ops.save_fl PUSHF
0x01 NOP6
...
0x07 NOP POP %RAX
The problem here is that offset-0x7, which is an instruction boundary in
both possible instruction patch streams, has two conflicting stack
layouts.
[ The above examples were stolen from Peter Zijlstra. ]
The new flattened CFI array is used both for the detection of conflicts
(like the second example above) and the generation of linear ORC
entries.
BTW, another benefit of these changes is that, thanks to some related
cleanups (new fake nops and alt_group struct) objtool can finally be rid
of fake jumps, which were a constant source of headaches.
[1] https://lkml.kernel.org/r/20201111170536.arx2zbn4ngvjoov7@treble
Cc: Shinichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2020-12-18 14:26:21 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Alternatives can have different stack layout
|
|
|
|
* possibilities (but they shouldn't conflict).
|
|
|
|
* Instead of traversing the instructions, use the
|
|
|
|
* alt_group's flattened byte-offset-addressed CFI
|
|
|
|
* array.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < alt_group_len(alt_group); i++) {
|
|
|
|
struct cfi_state *cfi = alt_group->cfi[i];
|
|
|
|
if (!cfi)
|
|
|
|
continue;
|
objtool: Introduce CFI hash
commit 8b946cc38e063f0f7bb67789478c38f6d7d457c9 upstream.
Andi reported that objtool on vmlinux.o consumes more memory than his
system has, leading to horrific performance.
This is in part because we keep a struct instruction for every
instruction in the file in-memory. Shrink struct instruction by
removing the CFI state (which includes full register state) from it
and demand allocating it.
Given most instructions don't actually change CFI state, there's lots
of repetition there, so add a hash table to find previous CFI
instances.
Reduces memory consumption (and runtime) for processing an
x86_64-allyesconfig:
pre: 4:40.84 real, 143.99 user, 44.18 sys, 30624988 mem
post: 2:14.61 real, 108.58 user, 25.04 sys, 16396184 mem
Suggested-by: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210624095147.756759107@infradead.org
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>
[bwh: Backported to 5.10:
- Don't use bswap_if_needed() since we don't have any of the other fixes
for mixed-endian cross-compilation
- Since we don't have "objtool: Rewrite hashtable sizing", make
cfi_hash_alloc() set the number of bits similarly to elf_hash_bits()
- objtool doesn't have any mcount handling
- Adjust context]
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-06-24 11:41:01 +02:00
|
|
|
/* errors are reported on the original insn */
|
|
|
|
if (init_orc_entry(&orc, cfi, insn))
|
objtool: Support stack layout changes in alternatives
commit c9c324dc22aab1687da37001b321b6dfa93a0699 upstream.
The ORC unwinder showed a warning [1] which revealed the stack layout
didn't match what was expected. The problem was that paravirt patching
had replaced "CALL *pv_ops.irq.save_fl" with "PUSHF;POP". That changed
the stack layout between the PUSHF and the POP, so unwinding from an
interrupt which occurred between those two instructions would fail.
Part of the agreed upon solution was to rework the custom paravirt
patching code to use alternatives instead, since objtool already knows
how to read alternatives (and converging runtime patching infrastructure
is always a good thing anyway). But the main problem still remains,
which is that runtime patching can change the stack layout.
Making stack layout changes in alternatives was disallowed with commit
7117f16bf460 ("objtool: Fix ORC vs alternatives"), but now that paravirt
is going to be doing it, it needs to be supported.
One way to do so would be to modify the ORC table when the code gets
patched. But ORC is simple -- a good thing! -- and it's best to leave
it alone.
Instead, support stack layout changes by "flattening" all possible stack
states (CFI) from parallel alternative code streams into a single set of
linear states. The only necessary limitation is that CFI conflicts are
disallowed at all possible instruction boundaries.
For example, this scenario is allowed:
Alt1 Alt2 Alt3
0x00 CALL *pv_ops.save_fl CALL xen_save_fl PUSHF
0x01 POP %RAX
0x02 NOP
...
0x05 NOP
...
0x07 <insn>
The unwind information for offset-0x00 is identical for all 3
alternatives. Similarly offset-0x05 and higher also are identical (and
the same as 0x00). However offset-0x01 has deviating CFI, but that is
only relevant for Alt3, neither of the other alternative instruction
streams will ever hit that offset.
This scenario is NOT allowed:
Alt1 Alt2
0x00 CALL *pv_ops.save_fl PUSHF
0x01 NOP6
...
0x07 NOP POP %RAX
The problem here is that offset-0x7, which is an instruction boundary in
both possible instruction patch streams, has two conflicting stack
layouts.
[ The above examples were stolen from Peter Zijlstra. ]
The new flattened CFI array is used both for the detection of conflicts
(like the second example above) and the generation of linear ORC
entries.
BTW, another benefit of these changes is that, thanks to some related
cleanups (new fake nops and alt_group struct) objtool can finally be rid
of fake jumps, which were a constant source of headaches.
[1] https://lkml.kernel.org/r/20201111170536.arx2zbn4ngvjoov7@treble
Cc: Shinichiro Kawasaki <shinichiro.kawasaki@wdc.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2020-12-18 14:26:21 -06:00
|
|
|
return -1;
|
|
|
|
if (!memcmp(&prev_orc, &orc, sizeof(orc)))
|
|
|
|
continue;
|
|
|
|
if (orc_list_add(&orc_list, &orc, insn->sec,
|
|
|
|
insn->offset + i))
|
|
|
|
return -1;
|
|
|
|
nr++;
|
|
|
|
prev_orc = orc;
|
|
|
|
empty = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Skip to the end of the alt_group */
|
|
|
|
insn = alt_group->last_insn;
|
2017-07-11 10:33:42 -05:00
|
|
|
}
|
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
/* Add a section terminator */
|
|
|
|
if (!empty) {
|
|
|
|
orc_list_add(&orc_list, &null, sec, sec->len);
|
|
|
|
nr++;
|
|
|
|
}
|
2017-07-11 10:33:42 -05:00
|
|
|
}
|
2020-12-17 15:02:42 -06:00
|
|
|
if (!nr)
|
|
|
|
return 0;
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
/* Create .orc_unwind, .orc_unwind_ip and .rela.orc_unwind_ip sections: */
|
|
|
|
sec = find_section_by_name(file->elf, ".orc_unwind");
|
|
|
|
if (sec) {
|
|
|
|
WARN("file already has .orc_unwind section, skipping");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
orc_sec = elf_create_section(file->elf, ".orc_unwind", 0,
|
|
|
|
sizeof(struct orc_entry), nr);
|
|
|
|
if (!orc_sec)
|
|
|
|
return -1;
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
sec = elf_create_section(file->elf, ".orc_unwind_ip", 0, sizeof(int), nr);
|
2017-12-30 14:43:32 -06:00
|
|
|
if (!sec)
|
|
|
|
return -1;
|
2017-07-11 10:33:42 -05:00
|
|
|
|
2020-12-17 15:02:42 -06:00
|
|
|
/* Write ORC entries to sections: */
|
|
|
|
list_for_each_entry(entry, &orc_list, list) {
|
2021-03-26 16:12:07 +01:00
|
|
|
if (write_orc_entry(file->elf, orc_sec, sec, idx++,
|
2020-12-17 15:02:42 -06:00
|
|
|
entry->insn_sec, entry->insn_off,
|
|
|
|
&entry->orc))
|
|
|
|
return -1;
|
2017-07-11 10:33:42 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|