Skip Unused Flags
Only computes the flags that later instructions actually read.
Where this happens
run in codegen/optimizations/eliminate_dead_flags.rs.
Summary
On x86, math instructions write flags that often nobody reads.
(Just like humans don't read LLM-generated garbage)
sub eax, 0 ; writes all flags
jz one ; reads ZF
dec eax ; writes all flags, except CF
jz three ; reads ZF
dec eax ; writes all flags, except CF
jnz one ; reads ZF
Every jump reads only ZF, and nothing reads CF.
So sub and each dec only need to compute ZF.
Computing each flag costs host instructions
So we can go more brr-brr if we skip the ones nobody reads.
Before and after
sub_59FE0 from Shadow the Hedgehog.
Reconstructed C++
Roughly what the original source looked like.
int sub_59FE0()
{
switch (field_38) // <- [ecx+0x38]
{
case 0: return 1;
case 1: return 3;
case 2: return 4;
default: return 1;
}
}
The switch was likely lowered to this before code generation.
int sub_59FE0()
{
int value = field_38; // mov eax, [ecx+0x38]
if (value == 0) return 1; // sub eax, 0 + jz
value -= 1; // dec eax
if (value == 0) return 3; // jz
value -= 1; // dec eax
if (value != 0) return 1; // jnz
return 4;
}
Recompiled C++
X86_PROTECTED_FUNC(sub_59FE0) {
// mov eax, [ecx+0x38]
ctx.eax.u32 = X86_LOAD_U32(ctx.ecx.u32 + 0x38); // <- field_38
// sub eax, 0x00
X86_SUB32(ctx.eax.u32, 0, ctx);
// jz 0x00059FFA
if (X86_CC_Z(ctx)) goto loc_59FFA;
// dec eax
X86_DEC32(ctx.eax.u32, ctx);
// jz 0x00059FF4
if (X86_CC_Z(ctx)) goto loc_59FF4;
// dec eax
X86_DEC32(ctx.eax.u32, ctx);
// jnz 0x00059FFA
if (X86_CC_NZ(ctx)) goto loc_59FFA;
// mov eax, 0x04
ctx.eax.u32 = 0x4u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FF4:
// mov eax, 0x03
ctx.eax.u32 = 0x3u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FFA:
// mov eax, 0x01
ctx.eax.u32 = 0x1u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
}
X86_PROTECTED_FUNC(sub_59FE0) {
// mov eax, [ecx+0x38]
ctx.eax.u32 = X86_LOAD_U32(ctx.ecx.u32 + 0x38); // <- field_38
// sub eax, 0x00
X86_SUB32<0x40>(ctx.eax.u32, 0, ctx); // <- ZF only
// jz 0x00059FFA
if (X86_CC_Z(ctx)) goto loc_59FFA;
// dec eax
X86_DEC32<0x40>(ctx.eax.u32, ctx);
// jz 0x00059FF4
if (X86_CC_Z(ctx)) goto loc_59FF4;
// dec eax
X86_DEC32<0x40>(ctx.eax.u32, ctx);
// jnz 0x00059FFA
if (X86_CC_NZ(ctx)) goto loc_59FFA;
// mov eax, 0x04
ctx.eax.u32 = 0x4u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FF4:
// mov eax, 0x03
ctx.eax.u32 = 0x3u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FFA:
// mov eax, 0x01
ctx.eax.u32 = 0x1u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
}
Its callers don't read flags after ret, so ZF is all we need.
As inlined into sub_5AAF0 by Clang.
X86_WEAK_FUNC(sub_5AAF0) {
// ...
loc_5AD91:
// mov ecx, esi
ctx.ecx.u32 = ctx.esi.u32;
// call 0x00059FE0
ctx.SubEsp(1); //Adjust stack pointer for call instruction
sub_59FE0(ctx, base); // <- inlined here
// pop edi
ctx.edi.u32 = X86_POP<false>(ctx, base);
// ...
}
Written by hand for comparison.
uint32_t sub_59FE0(uint32_t ecx, uint8_t* base)
{
uint32_t eax = X86_LOAD_U32(ecx + 0x38); // <- field_38
if (eax == 0) return 1;
eax -= 1;
if (eax == 0) return 3;
eax -= 1;
if (eax != 0) return 1;
return 4;
}
The After body, with ecx passed in and eax returned.
uint32_t sub_59FE0(uint32_t ecx, uint8_t* base)
{
X86Context ctx{}; // <- local
ctx.ecx.u32 = ecx;
// mov eax, [ecx+0x38]
ctx.eax.u32 = X86_LOAD_U32(ctx.ecx.u32 + 0x38); // <- field_38
// sub eax, 0x00
X86_SUB32<0x40>(ctx.eax.u32, 0, ctx);
// jz 0x00059FFA
if (X86_CC_Z(ctx)) goto loc_59FFA;
// dec eax
X86_DEC32<0x40>(ctx.eax.u32, ctx);
// jz 0x00059FF4
if (X86_CC_Z(ctx)) goto loc_59FF4;
// dec eax
X86_DEC32<0x40>(ctx.eax.u32, ctx);
// jnz 0x00059FFA
if (X86_CC_NZ(ctx)) goto loc_59FFA;
// mov eax, 0x04
ctx.eax.u32 = 0x4u;
return ctx.eax.u32;
loc_59FF4:
// mov eax, 0x03
ctx.eax.u32 = 0x3u;
return ctx.eax.u32;
loc_59FFA:
// mov eax, 0x01
ctx.eax.u32 = 0x1u;
return ctx.eax.u32;
}
The After body by hand, with branches reading eax instead of flags.
X86_PROTECTED_FUNC(sub_59FE0) {
// mov eax, [ecx+0x38]
ctx.eax.u32 = X86_LOAD_U32(ctx.ecx.u32 + 0x38); // <- field_38
// sub eax, 0x00
// jz 0x00059FFA
if (ctx.eax.u32 == 0) goto loc_59FFA; // <- no flags
// dec eax
ctx.eax.u32 -= 1;
// jz 0x00059FF4
if (ctx.eax.u32 == 0) goto loc_59FF4;
// dec eax
ctx.eax.u32 -= 1;
// jnz 0x00059FFA
if (ctx.eax.u32 != 0) goto loc_59FFA;
// mov eax, 0x04
ctx.eax.u32 = 0x4u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FF4:
// mov eax, 0x03
ctx.eax.u32 = 0x3u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
loc_59FFA:
// mov eax, 0x01
ctx.eax.u32 = 0x1u;
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
}
Host assembly
Clang 21.1.8, x86-64, -O2 -march=sandybridge.
| Version | Instructions |
|---|---|
| Before | 82 |
| After | 33 |
| After (Inlined) | 28 |
| Original | 13 |
| Hand-written | 13 |
| Param + local ctx | 13 |
| Merged branches | 16 |
Note that after inlining, the 3 instruction difference for Merged branches disappears; as Clang can optimise away the stack adjustment.
push rbp
mov rbp, rsp
mov eax, DWORD PTR [rdi+16] ; guest ecx
add eax, 56 ; + 0x38
mov eax, DWORD PTR [rsi+rax] ; load field_38
; sub eax, 0
xor ecx, ecx
test eax, eax
sete cl ; ZF
shl ecx, 6
mov edx, eax
shr edx, 24
and edx, -128 ; SF
xor esi, esi
test al, al
setnp sil ; PF
or edx, ecx
lea ecx, [rdx+4*rsi]
xor ecx, 4
mov DWORD PTR [rdi+84], ecx ; store guest EFLAGS
test eax, eax
je .LBB0_5 ; jz
; dec eax
xor edx, edx
mov ecx, eax
dec ecx
sete dl ; ZF
shl edx, 6
mov esi, ecx
shr esi, 24
and esi, -128 ; SF
xor r8d, r8d
test cl, cl
setnp r8b ; PF
xor r9d, r9d
mov r10d, eax
neg r10d
seto r9b ; OF
shl r9d, 11
mov r10d, ecx
xor r10d, eax
and r10d, 16 ; AF
or r10d, r9d
or r10d, esi
or r10d, edx
lea edx, [r10+4*r8]
xor edx, 4
mov DWORD PTR [rdi+84], edx ; store guest EFLAGS
test ecx, ecx
je .LBB0_2 ; jz
; dec eax
xor edx, edx
add eax, -2
sete dl ; ZF
shl edx, 6
mov esi, eax
shr esi, 24
and esi, -128 ; SF
xor r8d, r8d
test al, al
setnp r8b ; PF
xor r9d, r9d
mov r10d, ecx
neg r10d
seto r9b ; OF
shl r9d, 11
xor ecx, eax
and ecx, 16 ; AF
or ecx, r9d
or ecx, esi
or ecx, edx
lea ecx, [rcx+4*r8]
xor ecx, 4
mov DWORD PTR [rdi+84], ecx ; store guest EFLAGS
test eax, eax
je .LBB0_4 ; jnz (Clang flipped it)
.LBB0_5:
mov eax, 1
jmp .LBB0_6
.LBB0_2:
mov eax, 3
jmp .LBB0_6
.LBB0_4:
mov eax, 4
.LBB0_6:
mov DWORD PTR [rdi], eax ; guest eax
add DWORD PTR [rdi+56], 4 ; guest esp += 4
pop rbp
ret
push rbp
mov rbp, rsp
mov eax, DWORD PTR [rdi+16] ; guest ecx
add eax, 56 ; + 0x38
mov eax, DWORD PTR [rsi+rax] ; load field_38
; sub eax, 0
xor ecx, ecx
test eax, eax
sete cl ; ZF
shl ecx, 6
mov DWORD PTR [rdi+84], ecx ; store guest EFLAGS
test eax, eax
je .LBB0_3 ; jz
; dec eax
xor ecx, ecx
cmp eax, 1
sete cl ; ZF
shl ecx, 6
mov DWORD PTR [rdi+84], ecx ; store guest EFLAGS
mov ecx, 3
cmp eax, 1
je .LBB0_4 ; jz
; dec eax
xor ecx, ecx
cmp eax, 2
sete cl ; ZF
shl ecx, 6
mov DWORD PTR [rdi+84], ecx ; store guest EFLAGS
mov ecx, 4
cmp eax, 2
je .LBB0_4 ; jnz (Clang flipped it)
.LBB0_3:
mov ecx, 1
.LBB0_4:
mov DWORD PTR [rdi], ecx ; guest eax
add DWORD PTR [rdi+56], 4 ; guest esp += 4
pop rbp
ret
As inlined into sub_5AAF0 by Clang.
mov ecx, DWORD PTR [rbx+32] ; guest esi (from `mov ecx, esi`)
add ecx, 56 ; + 0x38
mov ecx, DWORD PTR [r14+rcx] ; load field_38
; sub eax, 0
xor edx, edx
test ecx, ecx
sete dl ; ZF
shl edx, 6
mov DWORD PTR [rbx+84], edx ; store guest EFLAGS
test ecx, ecx
je .LBB78_56 ; jz
; dec eax
xor edx, edx
cmp ecx, 1
sete dl ; ZF
shl edx, 6
mov DWORD PTR [rbx+84], edx ; store guest EFLAGS
mov edx, 3
cmp ecx, 1
je .LBB78_57 ; jz
; dec eax
xor edx, edx
cmp ecx, 2
sete dl ; ZF
shl edx, 6
mov DWORD PTR [rbx+84], edx ; store guest EFLAGS
mov edx, 4
cmp ecx, 2
je .LBB78_57 ; jnz (Clang flipped it)
.LBB78_56:
mov edx, 1
.LBB78_57:
mov DWORD PTR [rbx], edx ; guest eax
From the original .xbe.
mov eax, DWORD PTR [ecx+0x38] ; load field_38
sub eax, 0
jz loc_59FFA
dec eax
jz loc_59FF4
dec eax
jnz loc_59FFA
mov eax, 4
ret
loc_59FF4:
mov eax, 3
ret
loc_59FFA:
mov eax, 1
ret
push rbp
mov rbp, rsp
add edi, 56 ; ecx + 0x38
mov eax, DWORD PTR [rsi+rdi] ; load field_38
xor ecx, ecx
cmp eax, 1
sete cl
lea ecx, [2*rcx+1] ; 1 -> 3, else 1
cmp eax, 2
mov eax, 4
cmovne eax, ecx ; 2 -> 4
pop rbp
ret
push rbp
mov rbp, rsp
add edi, 56 ; ecx + 0x38
mov eax, DWORD PTR [rsi+rdi] ; load field_38
xor ecx, ecx
cmp eax, 1
sete cl
lea ecx, [2*rcx+1] ; 1 -> 3, else 1
cmp eax, 2
mov eax, 4
cmovne eax, ecx ; 2 -> 4
pop rbp
ret
push rbp
mov rbp, rsp
mov eax, DWORD PTR [rdi+16] ; guest ecx
add eax, 56 ; + 0x38
mov eax, DWORD PTR [rsi+rax] ; load field_38
xor ecx, ecx
cmp eax, 1
sete cl
lea ecx, [2*rcx+1] ; 1 -> 3, else 1
cmp eax, 2
mov eax, 4
cmovne eax, ecx ; 2 -> 4
mov DWORD PTR [rdi], eax ; guest eax
add DWORD PTR [rdi+56], 4 ; guest esp += 4
pop rbp
ret
How it works
Needed Flags is used to work out which flags are still needed after each instruction.
When we write the C++, these flags are passed as a mask:
X86_SUB32<0x40>(ctx.eax.u32, 0, ctx); // <- 0x40 is ZF
e.g.
<0x0>== no flags.- No mask -> all six (CF, PF, AF, ZF, SF and OF).
Calls and ret can (in theory) pass flags to other functions.
Emphasis on 'in theory', as in practice eflags is assumed to be clobbered. Only handrolled ASM assumes eflags is passed.