scripts/vgafixup.py

#!/usr/bin/env python
# Work around x86emu bugs by replacing problematic instructions.
#
# Copyright (C) 2012  Kevin O'Connor <kevin@koconnor.net>
#
# This file may be distributed under the terms of the GNU GPLv3 license.

# The x86emu code widely used in Linux distributions when running Xorg
# in vesamode is known to have issues with "retl", "leavel", "entryl",
# "leal", and some variants of "calll".  This code modifies those
# instructions that are known to be generated by gcc to avoid
# triggering the x86emu bugs.

# It is also known that the Windows vgabios emulator has issues with
# addressing negative offsets to the %esp register.  That has been
# worked around by not using the gcc parameter "-fomit-frame-pointer"
# when compiling.

import sys, re

# leal parameter regex - example string: -3(%edx,%eax,8), %eax
re_leal = re.compile(
    r'^\s*(?P<offset>[^(]*?)\s*'
    r'\(\s*(?P<base>[^,)]*?)\s*(?:,\s*(?P<index>[^,)]*?)\s*)?'
    r'(?:,\s*(?P<scale>[^,)]*?)\s*)?\)\s*'
    r',\s*(?P<dest>.*?)\s*$')

# Find an alternate set of instructions for a given "leal" instruction
def handle_leal(sline):
    m = re_leal.match(sline[5:])
    if m is None or m.group('index') == '%esp':
        print("Unable to fixup leal instruction: %s" % (sline,))
        sys.exit(-1)
    offset, base, index, scale, dest = m.group(
        'offset', 'base', 'index', 'scale', 'dest')
    if dest == '%esp':
        # If destination is %esp then just use 16bit leaw instead
        return 'leaw %s\n' % (sline[5:].replace('%e', '%'),)
    if not scale:
        scale = '1'
    scale = {1: 0, 2: 1, 4: 2, 8: 3}[int(scale, 0)]
    # Try to rearrange arguments to simplify 'base' (to improve code gen)
    if not scale and base == index:
        base, index, scale = '', index, 1
    elif not index or (not scale and base in (dest, '%esp') and index != dest):
        base, index, scale = index, base, 0
    # Produce instructions to calculate "leal"
    insns = ['pushfw']
    if base != dest:
        # Calculate "leal" directly in dest register
        if index != dest:
            insns.insert(0, 'movl %s, %s' % (index, dest))
        if scale:
            insns.append('shll $%d, %s' % (scale, dest))
        if base:
            if base == '%esp':
                offset += '+2'
            insns.append('addl %s, %s' % (base, dest))
    elif base == index:
        # Use "imull" method
        insns.append('imull $%d, %s' % ((1<<scale)+1, dest))
    else:
        # Backup/restore index register and do scaling in index register
        insns.append('pushl %s' % (index,))
        insns.append('shll $%d, %s' % (scale, index))
        insns.append('addl %s, %s' % (index, dest))
        insns.append('popl %s' % (index,))
    if offset and offset != '0':
        insns.append('addl $%s, %s' % (offset, dest))
    insns.append('popfw\n')
    return ' ; '.join(insns)

def main():
    infilename, outfilename = sys.argv[1:]
    infile = open(infilename, 'r')
    out = []
    for line in infile:
        sline = line.strip()
        if sline == 'ret':
            out.append('retw $2\n')
        elif sline == 'leave':
            out.append('movl %ebp, %esp ; popl %ebp\n')
        elif sline.startswith('call'):
            out.append('pushw %ax ; callw' + sline[4:] + '\n')
        elif sline.startswith('leal'):
            out.append(handle_leal(sline))
            #print("-> %s\n   %s" % (sline, out[-1].strip()))
        else:
            out.append(line)
    infile.close()
    outfile = open(outfilename, 'w')
    outfile.write(''.join(out))
    outfile.close()

if __name__ == '__main__':
    main()