scripts/vgafixup.py

#!/usr/bin/env python
# Work around x86emu bugs by replacing problematic instructions.
#
# Copyright (C) 2012  Kevin O'Connor <kevin@koconnor.net>
#
# This file may be distributed under the terms of the GNU GPLv3 license.

# The x86emu code widely used in Linux distributions when running Xorg
# in vesamode is known to have issues with "retl", "leavel", "entryl",
# and some variants of "calll".  This code modifies those instructions
# (ret and leave) that are known to be generated by gcc to avoid
# triggering the x86emu bugs.

# It is also known that the Windows vgabios emulator has issues with
# addressing negative offsets to the %esp register.  That has been
# worked around by not using the gcc parameter "-fomit-frame-pointer"
# when compiling.

import sys

def main():
    infilename, outfilename = sys.argv[1:]
    infile = open(infilename, 'rb')
    out = []
    for line in infile:
        sline = line.strip()
        if sline == 'ret':
            out.append('retw $2\n')
        elif sline == 'leave':
            out.append('movl %ebp, %esp ; popl %ebp\n')
        elif sline.startswith('call'):
            out.append('pushw %ax ; callw' + sline[4:] + '\n')
        else:
            out.append(line)
    infile.close()
    outfile = open(outfilename, 'wb')
    outfile.write(''.join(out))
    outfile.close()

if __name__ == '__main__':
    main()