Stack Argument Elimination
Passes stack arguments in emulated x86 registers, where it's safe.
Basically __fastcall at home.
Where this happens
Under codegen/optimizations/eliminate_stack_args/:
runinmod.rsruns the steps below.Called::findincalled.rschecks each called function (step 1).caller::rewrite_callsincaller.rsrewrites the calls (step 3).Called::rewriteincalled.rsrewrites each called function (step 4).
Rationale
Typically in x86 you pass arguments via the stack
or so is typical of the common __cdecl and __stdcall calling conventions.
; caller
push ecx ; write argument 0
call f
; f
mov eax, [esp+4] ; load argument 0 into eax
add eax, 0x14
ret
However, in recompiled code, this can be extra costly, as stack accesses are routed via ctx->esp, which lives behind a pointer and may not be mapped to a register.
So we try to rewrite the callers and callees to skip this stack traffic, and pass directly in registers.
You may have heard of the term __fastcall before
Which certainly has GCC and Microsoft variants... This is basically the same idea.
It applies when:
- We know every caller. See Known Callers.
- Stack Tracking can follow the callee from its start.
- The callee loads arguments right at the start. See Entry Loads.
- Nothing in the callee could read an argument write we remove.
Before and after
Callee
Recompiled C++
// push 0x00
X86_PUSH(ctx, base, 0x0);
// mov ecx, edi
ctx.ecx.u32 = ctx.edi.u32;
// call 0x0022D060
ctx.SubEsp(1); //Adjust stack pointer for call instruction
sub_22D060(ctx, base);
// mov ecx, edi
ctx.ecx.u32 = ctx.edi.u32;
// (rewritten) mov eax, 0x00
ctx.eax.u32 = 0x0u;
// call 0x0022D060
ctx.SubEsp(2); //Adjust stack pointer for call instruction
X86_EXTERN_FUNC(sub_22D060_fastcall);
sub_22D060_fastcall(ctx, base);
X86_PROTECTED_FUNC(sub_22D060) {
// mov eax, [esp+0x04]
ctx.eax.u32 = X86_LOAD_U32(ctx.esp.u32 + 0x4);
// mov [ecx+0x04], eax
X86_STORE_U32(ctx.ecx.u32 + 0x4, ctx.eax.u32);
// ret 0x04
ctx.esp.u32 += 0x8; //Adjust esp for return instruction
return;
}
X86_FUNC_IMPL(sub_22D060_fastcall) {
// mov [ecx+0x04], eax
X86_STORE_U32(ctx.ecx.u32 + 0x4, ctx.eax.u32);
// ret 0x04
ctx.esp.u32 += 0x8; //Adjust esp for return instruction
return;
}
X86_PROTECTED_FUNC(sub_22D060) {
ctx.eax.u32 = X86_LOAD_U32(ctx.esp.u32 + 0x4);
sub_22D060_fastcall(ctx, base);
}
Host assembly
Clang 21.1.8, x86-64, -O2.
mov eax, DWORD PTR [rbx+56] ; guest ESP
lea ecx, [rax-4]
mov DWORD PTR [r14+rcx], 0 ; push: store the parameter
mov ecx, DWORD PTR [rbx+40] ; guest EDI
mov DWORD PTR [rbx+16], ecx ; guest ECX
add eax, -8
mov DWORD PTR [rbx+56], eax ; update guest ESP
call sub_22D060
mov eax, DWORD PTR [rbx+40] ; guest EDI
mov DWORD PTR [rbx+16], eax ; guest ECX
mov DWORD PTR [rbx], 0 ; guest EAX = parameter
add DWORD PTR [rbx+56], -8 ; update guest ESP
call sub_22D060_fastcall
mov eax, DWORD PTR [rdi+56] ; guest ESP
lea ecx, [rax+4]
mov ecx, DWORD PTR [rsi+rcx] ; load the parameter
mov DWORD PTR [rdi], ecx ; guest EAX
mov edx, DWORD PTR [rdi+16] ; guest ECX
add edx, 4
mov DWORD PTR [rsi+rdx], ecx ; object store
add eax, 8
mov DWORD PTR [rdi+56], eax ; ret 4
mov eax, DWORD PTR [rdi] ; guest EAX already holds it
mov ecx, DWORD PTR [rdi+16] ; guest ECX
add ecx, 4
mov DWORD PTR [rsi+rcx], eax ; object store
add DWORD PTR [rdi+56], 8 ; ret 4
Caller
Recompiled C++
// push ecx
X86_PUSH(ctx, base, ctx.ecx.u32);
// call 0x00014D40
ctx.SubEsp(1); //Adjust stack pointer for call instruction
sub_14D40(ctx, base);
// add esp, 0x04
X86_ADD32(ctx.esp.u32, 4, ctx);
// push ecx
X86_PUSH(ctx, base, ctx.ecx.u32);
// (rewritten) mov eax, ecx
ctx.eax.u32 = ctx.ecx.u32;
// call 0x00014D40
X86_EXTERN_FUNC(sub_14D40_fastcall);
sub_14D40_fastcall(ctx, base);
// add esp, 0x04
X86_ADD32(ctx.esp.u32, 4, ctx);
X86_PROTECTED_FUNC(sub_14D40) {
// mov eax, [esp+0x04]
ctx.eax.u32 = X86_LOAD_U32(ctx.esp.u32 + 0x4);
// add eax, 0x14
X86_ADD32(ctx.eax.u32, 20, ctx);
// ret
ctx.esp.u32 += 0x4; //Adjust esp for return instruction
return;
}
X86_FUNC_IMPL(sub_14D40_fastcall) {
// add eax, 0x14
X86_ADD32(ctx.eax.u32, 20, ctx);
// ret
return;
}
X86_PROTECTED_FUNC(sub_14D40) {
ctx.eax.u32 = X86_LOAD_U32(ctx.esp.u32 + 0x4);
ctx.esp.u32 += 0x4;
sub_14D40_fastcall(ctx, base);
}
Host assembly
Clang 21.1.8, x86-64, -O2.
mov ecx, DWORD PTR [r14+rcx] ; mov ecx, [eax]
mov DWORD PTR [rbx+16], ecx ; guest ECX
mov DWORD PTR [r14+rax], ecx ; push: store the parameter
add eax, -4
mov DWORD PTR [rbx+56], eax ; update guest ESP
call sub_14D40
mov ecx, DWORD PTR [rbx] ; guest EAX, the result
mov eax, DWORD PTR [rbx+56] ; guest ESP, for add esp, 4
mov ecx, DWORD PTR [r14+rcx] ; mov ecx, [eax]
mov DWORD PTR [rbx+16], ecx ; guest ECX
mov DWORD PTR [r14+rax], ecx ; keep the stored parameter
mov DWORD PTR [rbx], ecx ; guest EAX = parameter
call sub_14D40_fastcall
mov ecx, DWORD PTR [rbx] ; guest EAX, the result
mov eax, DWORD PTR [rbx+56] ; guest ESP, for add esp, 4
mov eax, DWORD PTR [rdi+56] ; guest ESP
add eax, 4
mov ecx, DWORD PTR [rsi+rax] ; load the parameter
mov esi, ecx
add esi, 20 ; add eax, 0x14
mov DWORD PTR [rdi], esi ; guest EAX
; ... flags for add eax, 0x14
mov DWORD PTR [rdi+56], eax ; ret
mov eax, DWORD PTR [rdi] ; guest EAX already holds it
mov edx, eax
add edx, 20 ; add eax, 0x14
mov DWORD PTR [rdi], edx ; guest EAX
; ... flags for add eax, 0x14
Step by step
- Check each callee. See Stack Arguments and Entry Loads.
- Check every caller. See Known Callers and Call Setup.
- Rewrite the callers. Move each argument into its register right before the call, and remove the argument writes nothing reads.
- Rewrite the callees. Remove their entry loads. See Known Callers.
Soz, no more in-depth details. Time constraints.
Hopefully the helper pages are enough.