#! /usr/bin/env python3 # Identicalise 8086 programs using lDebug # 2024--2026 by E. C. Masloch # # Usage of the works is permitted provided that this # instrument is retained with the works, so that any entity # that uses the works is notified of this instrument. # # DISCLAIMER: THE WORKS ARE WITHOUT WARRANTY. import sys, pty, os, time, subprocess import fcntl, signal, pathlib, re, contextlib import math import codecs import contextlib import argparse from enum import Enum from datetime import datetime, timezone from collections import deque SCRIPTDIR = pathlib.Path(__file__).parent build_name = os.getenv('build_name', 'debug') machine = os.getenv('DEFAULT_MACHINE', 'qemu') try: if os.path.isfile(SCRIPTDIR / ("../ldebug/tmp/qemutest/l%s.img" % build_name)): os.symlink("../ldebug/tmp/qemutest/l%s.img" % build_name, SCRIPTDIR / ("l%s.img" % build_name)) except FileExistsError: pass try: if os.path.isfile(SCRIPTDIR / ("../ldebug/bin/l%s.com" % build_name)): os.symlink("../ldebug/bin/l%s.com" % build_name, SCRIPTDIR / ("l%s.com" % build_name)) except FileExistsError: pass try: if os.path.isfile(SCRIPTDIR / "../ldebug/bin/extlib.eld"): os.symlink("../ldebug/bin/extlib.eld", SCRIPTDIR / "extlib.eld") except FileExistsError: pass if machine == 'dosemu': dpmiavailable = 1 elif machine == 'qemu': dpmiavailable = 0 if "x" in build_name and os.path.isfile(SCRIPTDIR / "dpmi.img"): dpmiavailable = 1 class InvalidVariableException(Exception): __module__ = Exception.__module__ pass class ControlCException(Exception): __module__ = Exception.__module__ pass class VersionCommandNoException(Exception): __module__ = Exception.__module__ pass class LogicError(Exception): __module__ = Exception.__module__ pass class InputError(Exception): __module__ = Exception.__module__ pass class ListingError(InputError): __module__ = Exception.__module__ pass class EditError(Exception): __module__ = Exception.__module__ pass class ldebug_class(): def setUp(self): self.debug = int(os.getenv('DEBUG', '0'), 0) self.defaultsleepduration = \ float(os.getenv('test_sleepduration', '0.05')) self.addsleepduration = \ float(os.getenv('test_addsleepduration', '0.0')) machine = os.getenv('DEFAULT_MACHINE', 'qemu') build_name = os.getenv('build_name', 'debug') (self.m, s) = pty.openpty() self.sname = os.ttyname(s) os.close(s) if self.debug & 1: print("") print("sname=" + self.sname + " machine=" + machine + " build_name=" + build_name) if machine == 'dosemu': executable = os.getenv('DOSEMU', 'dosemu') self.dpmiavailable = 1 if "x" in build_name: self.dpmidir = "dpmitest\\" self.dpmiexe = None command = "%s -dumb -K \"%s\" -E " % (executable, SCRIPTDIR) \ + "\'l%s.com /c=\\\"r dco or= 4000\\\"\'" % build_name \ + " -I \'serial { com 2 device %s }\'" % self.sname, elif machine == 'qemu': dpmiimg = "" self.dpmiavailable = 0 executable = os.getenv('QEMU', 'qemu-system-i386') if "x" in build_name and os.path.isfile(SCRIPTDIR / "dpmi.img"): self.dpmidir = "b:" self.dpmiexe = "b:hdpmi32.exe -r" self.dpmiavailable = 1 dpmiimg = " -fdb \"%s/dpmi.img\"" % SCRIPTDIR command = "%s -fda \"%s/l%s.img\"" % (executable, SCRIPTDIR, build_name) \ + dpmiimg \ + " -boot order=a" \ + " -display none 2> /dev/null" \ + " -chardev serial,id=serial2,path=%s" % self.sname \ + " -serial null -serial chardev:serial2", else: os.close(self.m) raise InvalidVariableException("Invalid content of environment variable DEFAULT_MACHINE") if self.debug & 1: print(command) self.vmprocess = subprocess.Popen( command, shell=True, preexec_fn=os.setsid, stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) self.buffer = "" try: waitingseconds = 0 while "Enter KEEP to confirm.\r" not in self.buffer: if self.vmprocess.poll() is not None: print("vm stopped") self.tearDown() raise AssertionError("vm stopped") try: self.buffer += os.read(self.m, 1).decode('cp437') except OSError: waitingseconds += 1 if self.debug & 1: print("\rWaiting %s seconds" % waitingseconds, end='') time.sleep(1) if self.debug & 1 and waitingseconds: print("") time.sleep(0.2) os.write(self.m, b"keep\r") fl = fcntl.fcntl(self.m, fcntl.F_GETFL) fcntl.fcntl(self.m, fcntl.F_SETFL, fl | os.O_NONBLOCK) initialise_commands = \ os.getenv('ident86_initialise_commands', '') if initialise_commands != "": os.write(self.m, initialise_commands .replace(';', '\r').encode() + b'\r') except KeyboardInterrupt: print("^C (aborting vm)") self.tearDown() raise ControlCException from None def tearDown(self): if self.debug & 1: print("tear down called") self.buffer = "" os.write(self.m, b"q\r") time.sleep(0.05) if self.debug & 1: self.do_read(sleepduration = None) print(self.buffer) if self.vmprocess and self.vmprocess.poll() is None: if self.debug & 1: print("tear down terminates children") os.killpg(os.getpgid(self.vmprocess.pid), signal.SIGTERM) # self.vmprocess.terminate() self.vmprocess.wait() os.close(self.m) def do_read_single(self): try: input = os.read(self.m, 1).decode('cp437') except BlockingIOError: try: time.sleep(0.001) input = os.read(self.m, 1).decode('cp437') if self.debug & 1: print("success after sleeping") sys.stdout.flush() except BlockingIOError: input = None pass except OSError: input = None pass pass except OSError: input = None pass return input def do_read(self, sleepduration = 0.0, beepreplace = ''): if sleepduration is not None: if sleepduration != 0.0: time.sleep(sleepduration + self.addsleepduration) else: time.sleep(self.defaultsleepduration) overwrite = 0 linebuffer = "" repeat = 4 while repeat: input = self.do_read_single() if input: break if self.debug & 1: print("repeating") sys.stdout.flush() repeat -= 1 time.sleep(0.005) while input: if overwrite: if input != '\n': linebuffer = "" overwrite = 0 if input == '\r': overwrite = 1 if beepreplace is None: linebuffer += input else: linebuffer += input.replace('\x07', beepreplace) if input == '\n': self.buffer += linebuffer linebuffer = "" input = self.do_read_single() self.buffer += linebuffer def do_write_long(self, text, sleepduration=0.05, sleepmodulo=8): text = re.sub(r"[\r\n]+ +", "\r", text) text = re.sub(r"^\r+|\r+$", "", text) parts = text.split('\r') ii = 0 while parts: ii += 1 if (ii % sleepmodulo) == 0: if self.debug & 1: print("sleeping") time.sleep(sleepduration) part = parts.pop(0) if self.debug & 1: print("appending >>%s<<" % part) part += '\r' os.write(self.m, part.encode()) def inplace(orig_path, encoding='latin-1'): """Modify a file in-place, with a consistent encoding. From https://stackoverflow.com/questions/25203040/combining-inplace-filtering-and-the-setting-of-encoding-in-the-fileinput-module/28653349#28653349 """ while os.path.islink(orig_path): orig_path = str(pathlib.Path(orig_path).parent / os.readlink(orig_path)) new_path = orig_path + '.tmp' with codecs.open(orig_path, encoding=encoding) as orig: with codecs.open(new_path, 'w', encoding=encoding) as new: for line in orig: yield line, new os.rename(new_path, orig_path) # https://github.com/abergs/tabstospaces/issues/4#issuecomment-524465830 tabstopwidth = 8 def replacefunc(length, string): pattern = re.compile(r"^((?:[^\t]{%u})*[^\t]{%u})\t" % (tabstopwidth, length)) if match := re.search(pattern, string): replace = match.group(1) + " " * (tabstopwidth - length) string = re.sub(pattern, replace, string, count = 1) return string; def expandtabs(string): while "\t" in string: for length in range(0, tabstopwidth): string = replacefunc(length, string) return string def hex64bitstosigned(string): match = re.search(r'^[Ff]{8}([0-9A-Fa-f]{8})$', string) if match is not None: return - (0x100000000 - int(match.group(1), 16)) else: return int(string, 16) def ident86main(): SourcePosition = Enum('SourcePosition', ['afterprior', 'exactly', 'diff']) SourceEdit = Enum('SourceEdit', ['insertline', 'insertsegment', 'changesegment', 'removesegment', 'forcejumpnear', 'makefardd', 'wlcalc']) def readreloctable(file, offset, count, which): file.seek(offset, 0) table = file.read(count * 4) if not len(table) == count * 4: raise InputError("Unable to read relocation table") file.seek(0, 0) relocs = {} for idx in range(0, count * 4, 4): reloc = table[idx] + table[idx + 1] * 256 reloc += (table[idx + 2] + table[idx + 3] * 256) * 16 if reloc in relocs: raise InputError("File %u exactly overlap reloc %u at %06Xh!" % (which, idx // 4, reloc)) relocs[reloc] += 1 else: relocs[reloc] = 1 if reloc and (reloc - 1) in relocs: raise InputError(("File %u overlap reloc %u at %06Xh!" + " (have one before at %06Xh)") % (which, idx // 4, reloc, reloc - 1)) if (reloc + 1) in relocs: raise InputError(("File %u overlap reloc %u at %06Xh!" + " (have one after at %06Xh)") % (which, idx // 4, reloc, reloc + 1)) return relocs def getalloc(header): headersize = (header[8] + header[9] * 256) * 16 pages = header[4] + header[5] * 256 extrabytes = header[2] + header[3] * 256 minalloc = header[0xA] + header[0xB] * 256 if extrabytes > 512 or extrabytes == 0: extrabytes = 512 if not pages: raise InputError("Expected at least one page in MZ executable") imagesize = (pages - 1) * 512 + extrabytes - headersize if (pages - 1) * 512 + extrabytes <= headersize: raise InputError("Expected at least one byte in MZ executable image") return minalloc + (imagesize + 15) // 16 def addlistu(address, ustart, ulength): nonlocal mapranges nonlocal args global object ranges = () idx = 0 while idx < len(mapranges): assert ulength, "Loop should never be entered with empty ulength" if object.debug & (1 + 8192 * 1024): print("###", end = '') print(mapranges) print("a. idx=%u ustart=%06X ulength=%06X" % (idx, ustart + address, ulength)) print(mapranges[idx]) tuple = mapranges[idx] start = tuple[0] end = tuple[1] if object.debug & (1 + 8192 * 1024): print("b. idx=%u start=%06X end=%06X" % (idx, start, end)) if start <= ustart + address and end + 1 >= ustart + address + ulength: ranges += ([ustart, ulength, "u"],) return ranges if end < ustart + address: idx += 1 continue if start > ustart + address: rangelength = start - (ustart + address) # 12 - (0 + 10) = 2 rangelength = min(ulength, rangelength) # 2 ranges += ([ustart, rangelength, "b"],) # 0, 2, "b" ulength -= rangelength # 2 ustart += rangelength # 2 if not ulength: return ranges if object.debug & (1 + 8192 * 1024): print("c. ustart=%06X ulength=%06X" % (ustart + address, ulength)) rangelength = end + 1 - (ustart + address) # 13 + 1 - (2 + 10) = 2 rangelength = min(ulength, rangelength) # 2 ranges += ([ustart, rangelength, "u"],) # 2, 2, "u" ulength -= rangelength # 0 ustart += rangelength # 4 if not ulength: return ranges continue """ start = 12 end = 13 address = 10 ustart = 0 ulength = 4 want = 0, 2, "b" ustart = 4 - 2 = 2 end = 13 rangelength = end + 1 - (ustart + address) = 13 + 1 - (2 + 10) = 2 want = 2, 2, "u" """ assert False, "Unexpectedly did not return from loop" def disassemble(disbuffer, address, length, directives): global object dumphead = "\re cs:100" dump = '' modulo = 0 for byte in disbuffer: if (modulo % 8) == 0: dump += dumphead modulo += 1 dump += " %02X" % byte dumphead = "\re cs:aeo" dump += "\rh aeo\r" object.do_write_long(dump, sleepduration = 0.075, sleepmodulo = 2) object.do_read(sleepduration = 0.075) if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger output" object.buffer = "" list = () index = 0 ustart = 0 ulength = 0 linedisplacement = 0 if object.debug & 1: print("disassemble length=%u" % (length)) while linedisplacement < length: if object.debug & 1: # print(directives) for tuple in directives: if tuple[0] >= 0x8110 and tuple[0] <= 0x8130: print("disassemble directive tuple=", end="") print(tuple) print("linedisp=%u length=%u index=%u" \ % (linedisplacement, length, index)) tuple = None if index < len(directives): tuple = directives[index] if tuple is not None: if object.debug & 1: print(tuple) (start, amount, size) = tuple start -= address if object.debug & 32769: print("start=%u address=%u" % (start, address)) if tuple is not None: if start < 0: minus = start start = 0 amount = amount + minus if linedisplacement in range(start, start + amount): if ulength: list += addlistu(address, ustart, ulength) amount = min(amount, length - linedisplacement) assert start >= 0, \ ("Unexpected first tuple start=%d amount=%d size='%s'" \ + " linedisplacement=%d") \ % (start, amount, size, linedisplacement) if size == "w": amount += 1 amount &= ~1 if size == "d": amount += 3 amount &= ~3 list += ([start, amount, size],) linedisplacement += amount ustart = linedisplacement ulength = 0 index += 1 continue # skip increment elif linedisplacement >= start + amount: index += 1 continue # skip increment else: ulength += 1 else: ulength += 1 linedisplacement += 1 if ulength: list += addlistu(address, ustart, ulength) if object.debug & 1: print("list: ", end='') print(list) mergedlist = () priorstart = None prioramount = None priorsize = None assert len(list), "Expected at least one entry in disassembly list" for tuple in list: (start, amount, size) = tuple if priorstart is not None: if priorstart + prioramount == start and priorsize == size: prioramount += amount continue else: mergedlist += ([priorstart, prioramount, priorsize],) (priorstart, prioramount, priorsize) = tuple mergedlist += ([priorstart, prioramount, priorsize],) list = mergedlist if object.debug & 1: print("mergedlist: ", end='') print(mergedlist) entiretext = "" for tuple in list: (start, amount, size) = tuple assert start >= 0, "Unexpected tuple start=%d amount=%d size='%s'" \ % (start, amount, size) if size == "u": os.write(object.m, b"u cs:100 + %04X length %04X\r" \ % (start, amount)) object.do_read(sleepduration = 0.05 + (length // 32 * 0.05)) if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger output" text = object.buffer text = re.sub(r'u cs:100.*?[\r\n]+', '', text) text = re.sub(r'~?[-#]$', '', text) elif size == "b" or size == "w" or size == "d": os.write(object.m, ("d%s cs:100 + %04X length %04X\r" \ % (size, start, amount)).encode()) object.do_read(sleepduration = 0.05 + (length // 128 * 0.05)) if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger output" text = object.buffer text = re.sub(r'd[bwd] cs:100.*?[\r\n]+', '', text) text = re.sub(r'~?[-#]$', '', text) if size == "b": width = 2 # 2 digits perline = 16 elif size == "w": width = 4 # 4 digits perline = 8 elif size == "d": width = 8 # 8 digits perline = 4 datalinepattern = re.compile(r"""(?mix) ^([0-9A-Fa-f]{4}:)([0-9A-Fa-f]{4,8}) # $1, $2 = address \s\s(\s{0,%u}) # $3 = padding ( (?:(?:[0-9A-Fa-f]{%u}|\s{%u})\s){%u} # $4 = data as hex (?:(?:[0-9A-Fa-f]{%u}|\s{%u})\-){%u} # data as hex (?:(?:[0-9A-Fa-f]{%u}|\s{%u})\s){%u} # data as hex ) .*?[\r\n]*$ # data as text """ % (width - 2, # b = 0, w = 2, d = 6 width, width, perline // 2 - 1, width, width, 1, width, width, perline // 2)) while True: match = re.search(datalinepattern, text) if match is None: break datasegment = match.group(1) dataoffset = int(match.group(2), 16) padding = len(match.group(3)) dataoffset += padding // 2 # b = 0, w = 0..1, d = 0..3 data = match.group(4) datablankitempattern = re.compile(r"""(?xi) ^\s{%s}[\-\s]""" % width) dataitempattern = re.compile(r"""(?xi) ^([0-9A-Fa-f]{%s})[\-\s]""" % width) while True: match = re.search(datablankitempattern, data) if match is None: break dataoffset += width // 2 # b = 1, w = 2, d = 4 data = re.sub(datablankitempattern, '', data, count=1) replace = "" while True: match = re.search(dataitempattern, data) if match is None: break item = int(match.group(1), 16) replace += ("%s%08X " % (datasegment, dataoffset)) # address replace += "0" * width # placeholder for hexdump (stripped later) replace += " d" + size + (" %%0%uX" % width) % item # instruction replace += "\r\n" dataoffset += width // 2 # b = 1, w = 2, d = 4 data = re.sub(dataitempattern, '', data, count=1) text = re.sub(datalinepattern, replace, text, count=1) else: assert False, "Unknown size=%s in disassemble loop" % size object.buffer = "" entiretext += text text = entiretext text = re.sub(r'near +', '', text) patternmultiline = re.compile(r"""(?xi) (^|[\r\n]+) ([0-9A-Fa-f]{4}) : ([0-9A-Fa-f]{4,8}) (\s+) ((?:[0-9A-Fa-f]{2})+) [\r\n]+ \s{10,14} ((?:[0-9A-Fa-f]{2})+) """) while True: match = patternmultiline.search(text) if match is None: break; text = re.sub(patternmultiline, r'\1\2:\3\4\5\6', text, count = 1) pattern = re.compile(r'(?m)(?:^#SKIPME#)?[0-9A-Fa-f]{4}:([0-9A-Fa-f]{4,8}) +((?:[0-9A-Fa-f]{2})+) +') while True: match = pattern.search(text) if match is None: break; debuggeroffset = int(match.group(1), 16) assert debuggeroffset is not None, "Debugger output found no offset" assert debuggeroffset >= 0x100, "Debugger output unexpected offset" instructionlength = len(match.group(2)) // 2 instructionaddress = ("%06X +%X " % (debuggeroffset - 0x100 + address, instructionlength)) text = re.sub(pattern, instructionaddress, text, count = 1) text = re.sub(r'modrm +', '', text) text = re.sub(r'\[needs .*?\]', '', text) text = re.sub(r' *[\r\n]+', '\n', text) text = re.sub(r"""(?xi) (\[[^\]]*) (?:byte|word|dword)\s+([0-9A-Fa-f]{2,8}\]) """, r'\1\2', text) text = re.sub(r"""(?xi) ((?:mov|add|adc|sub|sbb|xor|and|or|test|cmp) \s+(?:word\s(?:ptr\s)?[^,]+|[a-d]x|[sd]i|[bs]p),\s*) (?:word\s)?([0-9A-Fa-f]{4})$ """, r'\1\2', text) while True: search = re.search(r"""(?xis) ^(.*(?:mov|add|adc|sub|sbb|xor|and|or|test|cmp) \s+(?:word\s(?:ptr\s)?[^,]+|[a-d]x|[sd]i|[bs]p),\s*) (?:word\s)\+?([0-9A-Fa-f]{4}.*)$ """, text) if search is None: break text = search.group(1) + search.group(2) while True: search = re.search(r"""(?xis) ^(.*(?:mov|add|adc|sub|sbb|xor|and|or|test|cmp) \s+(?:word\s(?:ptr\s)?[^,]+|[a-d]x|[sd]i|[bs]p),\s*) \-([0-9A-Fa-f]{2})(.*)$ """, text) if search is None: break minus = int(search.group(2), 16) minus = - minus; minus = ("%04X" % (minus & 0xFFFF)) text = search.group(1) + minus + search.group(3) while True: search = re.search(r"""(?xis) ^(.*(?:mov|add|adc|sub|sbb|xor|and|or|test|cmp) \s+(?:word\s(?:ptr\s)?[^,]+|[a-d]x|[sd]i|[bs]p),\s*) \+([0-9A-Fa-f]{2}.*)$ """, text) if search is None: break text = search.group(1) + "00" + search.group(2) while True: pattern = re.compile(r"""(?xi) (\[(?:[descfg]s\s*:\s*)? (?:bx|bx\s*\+\s*si|bx\s*\+\s*di|bp|bp\s*\+\s*si|bp\s*\+\s*di)\s*) \-(\s*)([0-9A-Fa-f]{2})(\]) """) search = re.search(pattern, text) if search is None: break minus = int(search.group(3), 16) minus = - minus; minus = ("%04X" % (minus & 0xFFFF)) replace = search.group(1) + "+" + search.group(2) + minus + search.group(4) text = re.sub(pattern, replace, text, count=1) while True: pattern = re.compile(r"""(?xi) (\[(?:[descfg]s\s*:\s*)? (?:bx|bx\s*\+\s*si|bx\s*\+\s*di|bp|bp\s*\+\s*si|bp\s*\+\s*di)\s*) \+(\s*)([0-9A-Fa-f]{2}\]) """) search = re.search(pattern, text) if search is None: break replace = search.group(1) + "+" + search.group(2) + "00" + search.group(3) text = re.sub(pattern, replace, text, count=1) while True: pattern = re.compile(r"""(?xi) (\[(?:[descfg]s\s*:\s*)? (?:bx|bx\s*\+\s*si|bx\s*\+\s*di|bp|bp\s*\+\s*si|bp\s*\+\s*di)) \s*\+\s*0{4}(\]) """) search = re.search(pattern, text) if search is None: break replace = search.group(1) + search.group(2) text = re.sub(pattern, replace, text, count=1) """ ((mov|add|adc|sub|sbb|xor|and|or|test|cmp| rol|ror|rcl|rcr|shl|sal|shr|sar| xchg|lds|les|lss|lfs|lgs)\s+) """ text = re.sub(r'\n+$', '', text) assert "^ Error" not in object.buffer, "Error in debugger output" object.buffer = "" return text def replacenumeric(text): pattern = re.compile(r"""(?xi) ( ^[0-9A-Fa-f]{6} # address \s\+[0-9A-Fa-f]+ # length \s+(?:(?:o32|o16|a32|a16|[descfg]s|repn?e?|lock)\s+)* # prefix (?:[0-9A-Za-z]+\s+) # mnemonic .*?) # prefix (\b[0-9A-Fa-f]+\b) # numeric """) numbers = () letter = "Z" while True: match = pattern.search(text) if match is None: break assert ord(letter) > ord("F"), "Replacement would clash with hexadecimal number" numbers += (int(match.group(2), 16),) replace = match.group(1) replace += letter * len(match.group(2)) text = re.sub(pattern, replace, text, count=1) letter = chr(ord(letter) - 1) return (text, numbers) def fuzzycompare(line1, line2): global object nonlocal args patternaddress = re.compile(r'^([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) (.*)') match1 = patternaddress.search(line1) match2 = patternaddress.search(line2) assert match1 is not None and match2 is not None, "Disassembly text does not match expected format" if match1.group(1) != match2.group(1): return False (line1, numbers1) = replacenumeric(line1) (line2, numbers2) = replacenumeric(line2) if len(numbers1) != len(numbers2): return False match1 = patternaddress.search(line1) match2 = patternaddress.search(line2) assert match1 is not None and match2 is not None, "Replaced text does not match expected format" if match1.group(3) != match2.group(3): return False idx = 0 while idx < len(numbers1): if abs(numbers1[idx] - numbers2[idx]) > int(args.fuzzy, 0): return False idx += 1 if object.debug & 1: print("Fuzzy compare equal \"%s\"" % line1) return True def swapoperands(line1, line2): """ if not fuzzycompare(line1, line2): pattern = re.compile( r'^(([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) ' + r'(test|xchg)\s+)(.*)(,\s+)(.*)$') # $1 = prefix, $6 = comma # $2 = address, $3 = length, $4 = inst, $5 = first, $7 = second match1 = pattern.search(line1) match2 = pattern.search(line2) if match1 is not None and match2 is not None \ and match1.group(2) == match2.group(2) \ and match1.group(4) == match2.group(4): line3 = match2.group(1) + match2.group(7) \ + match2.group(6) + match2.group(5) if fuzzycompare(line1, line3): line2 = line3 """ if line1 != line2: pattern = re.compile( r'^(([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) ' + r'(test|xchg)\s+)(.*)(,\s+)(.*)$') # $1 = prefix, $6 = comma # $2 = address, $3 = length, $4 = inst, $5 = first, $7 = second match1 = pattern.search(line1) match2 = pattern.search(line2) if match1 is not None and match2 is not None \ and match1.group(2) == match2.group(2) \ and match1.group(4) == match2.group(4) \ and match1.group(5) == match2.group(7) \ and match1.group(7) == match2.group(5): line2 = match2.group(1) \ + match2.group(7) + match2.group(6) + match2.group(5) return line2 def markearliest(line, offset, iswlcalc): nonlocal earliestdifferenceflag, args nonlocal maximum if iswlcalc and not args.wlcalc_replace: return if not iswlcalc and args.wlcalc_replace: return if earliestdifferenceflag is None: match = re.search('^([0-9A-Fa-f]{6}) ', line) assert match is not None, "Disassembly text does not match expected format" addressdifference = int(match.group(1), 16) earliestdifferenceflag = addressdifference print("Earliest difference!") if args.cookie != '': print("Writing 0x%06X to cookie" % offset) with open(args.cookie, 'a+') as cookiefile: cookiefile.seek(0, 0) lastline = None for line in cookiefile: lastline = line if lastline is not None and lastline != '' \ and int(lastline, 0) == offset: print("Cookie is already current!") else: cookiefile.write("0x%06X\n" % offset) if args.difference_length != "0": maximum = \ addressdifference + int(args.difference_length, 0) assert maximum > 0, "Difference length plus difference must be > 0, got %d" % maximum args.maximum_offset = \ ("0x%06X" % maximum); def handlenops(lines1, index1, lines2, index2): global object isfuzzy = False if lines1[index1] != lines2[index2]: pattern = re.compile(r'^([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) (.*)') patternjmp = re.compile(r'^jmp\b') match1 = pattern.search(lines1[index1]) match2 = pattern.search(lines2[index2]) assert match1 is not None and match2 is not None, \ "Disassembly text does not match expected format" patternxchgself = re.compile( r'^xchg\s+(e?[abcd][hlx]|e?[sd]i|e?[bs]p)\s*,\s*\1$') group3_1 = match1.group(3) group3_2 = match2.group(3) group3_1 = re.sub(patternxchgself, "nop", group3_1, count = 1) group3_2 = re.sub(patternxchgself, "nop", group3_2, count = 1) if group3_1 == group3_2: pass elif not fuzzycompare(lines1[index1], lines2[index2]): return (0, 0, False, None) length1 = int(match1.group(2), 16) length2 = int(match2.group(2), 16) if length1 == length2: return (0, 0, False, None) if match1.group(3) != match2.group(3): isfuzzy = True longer = 1 if length2 > length1: longer = 2 expectednopsamount = (length1, length2)[longer - 1] \ - (length2, length1)[longer - 1] expectednopsline = (lines2, lines1)[longer - 1] expectednopsindex = (index2, index1)[longer - 1] expectednopsaddress = int((match2, match1)[longer - 1].group(1), 16) # (longer - 1) ^ 1 = longer2 = 0, longer1 = 1 # longer - 1 = longer1 = 0, longer2 = 1 if length1 == 2 + ((longer - 1) ^ 1) \ and length2 == 2 + (longer - 1): if object.debug & 5: print("jmp check") matchjmp1 = patternjmp.search(match1.group(3)) matchjmp2 = patternjmp.search(match2.group(3)) if matchjmp1 is not None and matchjmp2 is not None: if object.debug & 5: print("jmp check is jmp") matchnop = pattern.search(expectednopsline[expectednopsindex + 1]) if matchnop is None or matchnop.group(3) != "nop": # longer ^ 3 = longer1 = 2, longer2 = 1 if args.edit_file == 0 or args.edit_file == longer ^ 3: print("hint: force jump to be near in file %u" \ % (longer ^ 3)) editsource(findsourceline(expectednopsaddress, SourcePosition.exactly), SourceEdit.forcejumpnear) else: print("hint: near jump needs to be short in file %u" \ % (longer)) return (0, 0, False, None) expectednopsaddress += (length2, length1)[longer - 1] while expectednopsamount: if object.debug & 9: print("nops check") expectednopsindex += 1 matchnop = pattern.search(expectednopsline[expectednopsindex]) assert matchnop is not None, \ "Disassembly text does not match expected format" if expectednopsaddress != int(matchnop.group(1), 16): break if matchnop.group(2) != "1": break if matchnop.group(3) != "nop": break expectednopsaddress += 1 expectednopsamount -= 1 if expectednopsamount: plural = "" if expectednopsamount != 1: plural = "s" if args.edit_file == 0 or args.edit_file == longer ^ 3: print("hint: insert %u NOP%s in file %u" \ % (expectednopsamount, plural, longer ^ 3)) editsource(findsourceline(expectednopsaddress, SourcePosition.afterprior), SourceEdit.insertline, insert="nop", amount = expectednopsamount) else: print("hint: instruction too long by %u byte%s in file %u" \ % (expectednopsamount, plural, longer)) return (0, 0, False, None) if object.debug & 9: print("nops match n=%u 1=%u 2=%u" % (expectednopsindex, index1, index2)) if longer == 1: return (0, expectednopsindex - index2, True, isfuzzy) else: return (expectednopsindex - index1, 0, True, isfuzzy) else: return (0, 0, True, False) def separatesegmentoverride(line): segaddrpattern = re.compile(r"""(?ix) (\b[descfg]s\b): """) segprefixpattern = re.compile(r"""(?ix) (\b[descfg]s\b) \s+(a32|a16|lods|stos|scas|cmps|movs|ins|outs|xlatb) """) matchsegaddr = re.search(segaddrpattern, line) matchsegpre = re.search(segprefixpattern, line) segment = None if matchsegaddr is not None: segment = matchsegaddr.group(1) line = re.sub(segaddrpattern, '', line, count=1) elif matchsegpre is not None: segment = matchsegpre.group(1) line = re.sub(segprefixpattern, r'\2', line, count=1) return (segment, line) def checkfardd(line1, line2): global object if object.debug & 129: print("line1=>%s< line2=>%s<" % (line1, line2)) ddsegmentpattern = re.compile(r"""(?ix) ^[0-9A-Fa-f]{6}\ \+4\ dd\ ([0-9A-Fa-f]{4})[0-9A-Fa-f]{4}$ """) dddropsegmentpattern = re.compile(r"""(?ix) ^([0-9A-Fa-f]{6}\ \+4\ dd\ )[0-9A-Fa-f]{4}([0-9A-Fa-f]{4})$ """) match1 = re.search(ddsegmentpattern, line1) match2 = re.search(ddsegmentpattern, line2) if match1 is None or match2 is None: return False line1 = re.sub(dddropsegmentpattern, r'\1\2', line1, count=1) line2 = re.sub(dddropsegmentpattern, r'\1\2', line2, count=1) if not fuzzycompare(line1, line2): return False if match1.group(1) == "0000" and match2.group(1) != "0000": if args.edit_file == 0 or args.edit_file == 1: print("hint: make dd directive a far pointer in file 1") match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.makefardd) else: print("hint: make far dd directive a near pointer in file 2") elif match1.group(1) != "0000" and match2.group(1) == "0000": if args.edit_file == 0 or args.edit_file == 2: print("hint: make dd directive a far pointer in file 2") match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.makefardd) else: print("hint: make far dd directive a near pointer in file 1") return False def checksegmentoverride(line1, line2): global object if object.debug & 1: print("line1=>%s< line2=>%s<" % (line1, line2)) (segment1, line1) = separatesegmentoverride(line1) (segment2, line2) = separatesegmentoverride(line2) if object.debug & 1: print("line1=>%s< line2=>%s<" % (line1, line2)) if not fuzzycompare(line1, line2): return False if segment1 is None and segment2 is None: return False if segment1 is None: if args.edit_file == 0 or args.edit_file == 1: print("hint: insert %s segment override in file 1" % segment2) match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.insertsegment, insert=segment2) else: print("hint: remove %s segment override in file 2" % segment2) match = re.search(r'^([0-9A-F]{6}) ', line2) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.removesegment, remove=segment2) elif segment2 is None: if args.edit_file == 0 or args.edit_file == 2: print("hint: insert %s segment override in file 2" % segment1) match = re.search(r'^([0-9A-F]{6}) ', line2) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.insertsegment, insert=segment1) else: print("hint: remove %s segment override in file 1" % segment1) match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.removesegment, remove=segment1) else: if args.edit_file == 0: print("hint: change %s (file 1) or %s (file 2) segment override" \ % (segment1, segment2)) elif args.edit_file == 1: print("hint: change %s (file 1) to %s segment override" \ % (segment1, segment2)) match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.changesegment, remove=segment1, insert=segment2) elif args.edit_file == 2: print("hint: change %s (file 2) to %s segment override" \ % (segment2, segment1)) match = re.search(r'^([0-9A-F]{6}) ', line1) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.exactly), SourceEdit.changesegment, remove=segment2, insert=segment1) return True def editsource(tuple, edittype, insert = None, remove = None, amount = 1): nonlocal args, edited global object if not args.do_edit: return (file, targetlinenumber) = tuple if edited: return if file is None: if args.repeat: raise EditError("File to edit not found!") return if object.debug & 17: print(("file=\"%s\" line=%u edittype=%s " + "insert=%s remove=%s amount=%u") \ % (file, targetlinenumber, edittype.name, insert, remove, amount)) patterneol = re.compile(r"([\r\n]+)$") eol = None done = 0 currentlinenumber = 0 for line, new in inplace(file): currentlinenumber += 1 matcheol = re.search(patterneol, line) if matcheol is not None: eol = matcheol.group(1) if currentlinenumber != targetlinenumber: if not done: if object.debug & 17: print("unchanged=%u >%s<" % (currentlinenumber, line.strip())) new.write(line) continue if object.debug & 17: print("to change=%u >%s<" % (currentlinenumber, line.strip())) done = 1 if edittype == SourceEdit.insertline: if eol is None: eol = "\n" for repeat in range(0, amount): new.write("\t" + insert + "\t; identicalise" + eol) elif edittype == SourceEdit.insertsegment: patternbracket = re.compile(r"^([^\]\[\;\r\n]+\[)") matchbracket = re.search(patternbracket, line) if matchbracket is not None: line = re.sub(patternbracket, r"\1" + insert + ":", line, count = 1) else: pattern = re.compile(r"""(?ix) ^(\s*[A-Za-z\._\$][A-Za-z\._\$0-9]*(?:\s+|:))?(\s*) ((?:repn?[ez]?\s+|a32\s+|a16\s+)* (?:(?:lods|stos|movs|cmps|scas|ins|outs)[bwd]|xlat|xlatb) )\b """) if not re.search(pattern, line): raise EditError("Instruction for segment override insertion not found") line = re.sub(pattern, r"\1\2" + insert + r" \3", line, count = 1) elif edittype == SourceEdit.changesegment: patternbracket = re.compile(r"(?i)^([^\]\[\;\r\n]+\[\s*)" + remove + r"(\s*:)") matchbracket = re.search(patternbracket, line) if matchbracket is not None: line = re.sub(patternbracket, r"\1" + insert + r"\2", line, count = 1) else: pattern = re.compile(r"""(?ix) ^(\s*[A-Za-z\._\$][A-Za-z\._\$0-9]*(?:\s+|:))?(\s*) ((?:repn?[ez]?\s+|a32\s+|a16\s+|""" + remove + r"""\s+)* (?:(?:lods|stos|movs|cmps|scas|ins|outs)[bwd]|xlat|xlatb) )\b """) if not re.search(pattern, line): raise EditError("Segment override to change not found") patternsegment = re.compile(r"(?i)\b" + remove + r"\b") if not re.search(patternsegment, line): raise EditError("Segment override to change not found") line = re.sub(patternsegment, insert, line, count = 1) elif edittype == SourceEdit.removesegment: patternbracket = re.compile(r"(?i)^([^\]\[\;\r\n]+\[\s*)" + remove + r"(\s*:)") matchbracket = re.search(patternbracket, line) if matchbracket is not None: line = re.sub(patternbracket, r"\1", line, count = 1) else: pattern = re.compile(r"""(?ix) ^(\s*[A-Za-z\._\$][A-Za-z\._\$0-9]*(?:\s+|:))?(\s*) ((?:repn?[ez]?\s+|a32\s+|a16\s+|""" + remove + r"""\s+)* (?:(?:lods|stos|movs|cmps|scas|ins|outs)[bwd]|xlat|xlatb) )\b """) if not re.search(pattern, line): raise EditError("Segment override to remove not found") patternsegment = re.compile(r"(?i)\b" + remove + r"\b\s*") if not re.search(patternsegment, line): raise EditError("Segment override to remove not found") line = re.sub(patternsegment, "", line, count = 1) elif edittype == SourceEdit.forcejumpnear: patternjump = re.compile(r"""(?ix) ^((?:\s*[A-Za-z\._\$\@][A-Za-z\._\$\@0-9]*(?:\s+|:))?\s* jmp\s+ )(?:(?:strict\s+)?short\s+)? """) if not re.search(patternjump, line): raise EditError("Jump to force near not found") line = re.sub(patternjump, r"\1strict near ", line, count = 1) elif edittype == SourceEdit.makefardd: patterndd = re.compile(r"""(?ix) ^((?:\s*[A-Za-z\._\$\@][A-Za-z\._\$\@0-9]*(?:\s+|:))?\s* )(dd)(\s+ )([A-Za-z\$\.\@_][0-9A-Za-z\$\.\@_]*) (\s*(?:;.*)?)$ """) if not re.search(patterndd, line): raise EditError("Directive dd to force far not found") line = re.sub(patterndd, r"\1dw\3\4, seg \4\5", line, count = 1) elif edittype == SourceEdit.wlcalc: # liner = re.sub(r'[\r\n]+', '', line) # print(">>"+liner+"<<") pattern1operand = re.compile(r"""(?ix) ^(\s*(?:db|dw|dd|repeatdw|neg|not|inc|dec|i?mul|i?div|push|pop|jmp|call|int|ret[nf]?)\s+) (.*[\r\n]*)$ """) patternshiftrot = re.compile(r"""(?ix) ^(\s*(?:shl|shr|sal|sar|rol|ror|rcl|rcr)\s+) (.*[\r\n]*)$ """) patternother = re.compile(r"""(?ix) ^(\s*[A-Za-z0-9]+\s+) (.*[\r\n]*)$ """) patternkeyword = re.compile(r'(?i)^(\s*(?:strict|byte|word|dword|near|far)\b\s*)(.*[\r\b\n]*)$') patternbracket = re.compile(r'(?i)^(\s*\[\s*)(.*[\r\b\n]*)$') patternimplicit = re.compile(r'(?i)([^,;]+)(,\s*(?:cl|1)\s*(?:;.*)?)$') patternsecondreg = re.compile(r'(?i)([^,;]+)(,\s*(?:e?[a-d][xhl]|e?[sd]i|e[sb]p|[desc]s)\s*(?:;.*)?)$') patternsecondoperand = re.compile(r'(?i)([^,;]+,)(.*[\r\b\n]*)$') oroginalline = line donewlcalc = False match = re.search(pattern1operand, line) def editwlcalc(): nonlocal match, prefixline, subline, line, args, donewlcalc while True: matchkeyword = re.search(patternkeyword, subline) if matchkeyword is None: break prefixline += matchkeyword.group(1) subline = matchkeyword.group(2) while True: matchbracket = re.search(patternbracket, subline) if matchbracket is None: break prefixline += matchbracket.group(1) subline = matchbracket.group(2) while True: matchkeyword = re.search(patternkeyword, subline) if matchkeyword is None: break prefixline += matchkeyword.group(1) subline = matchkeyword.group(2) blankbefore = ' ' if re.search(r'\s+$', prefixline): blankbefore = '' blankafter = ' ' if re.search(r'^\s+', subline): blankafter = '' line = match.group(1) + prefixline + blankbefore + 'relocatedzero +' + blankafter + subline donewlcalc = True if match is not None: subline = match.group(2) prefixline = '' editwlcalc() match = None if not donewlcalc: match = re.search(patternshiftrot, line) if match is not None: matchimplicit = re.search(patternimplicit, match.group(2)) if matchimplicit is not None: subline = match.group(2) prefixline = '' editwlcalc() match = None if not donewlcalc: match = re.search(patternother, line) if match is not None: matchsecondreg = re.search(patternsecondreg, match.group(2)) if matchsecondreg is not None: subline = match.group(2) prefixline = '' editwlcalc() else: matchsecondoperand = re.search(patternsecondoperand, match.group(2)) if matchsecondoperand is None: # print(">>"+match.group(2)+"<<") raise EditError("Replacing wlcalc failed") subline = matchsecondoperand.group(2) prefixline = matchsecondoperand.group(1) editwlcalc() if not donewlcalc: raise EditError("Replacing wlcalc failed") else: assert False, "Unknown source edit type passed down" # if not edittype == SourceEdit.wlcalc: edited = True new.write(line) if edited: print("Edited file \"%s\"" % file) if args.wlcalc_quick: args.maximum_offset = ("0") else: if args.repeat: raise EditError("Line to edit not found!") def highlightdump(string, differences): if not args.show_dump_colour: return string pattern = re.compile(r"""(?xi) ^((?:second:|first:)?\s*) # $1 = prefix ([0-9A-Fa-f]{6}) # $2 = start (\s+) # $3 = blank (\+) # $4 = plus sign ([0-9]*[1-9][0-9]*)\b # $5 = length (.*)$ # $6 = remainder """) match = re.search(pattern, string) if match is None: return string; start = int(match.group(2), 16) end = start + int(match.group(5), 10) if object.debug & 32769: print(match) print("s=%06Xh e=%06Xh" % (start, end)) for position in range(start, end): if object.debug & 32769: print("p=%06Xh" % (position)) if position in differences: if object.debug & 32769: print("MATCH") string = re.sub(pattern, "\\1\\2\\3\x1B[7m\\4\\5\x1B[m\\6", string, count = 1) return string; return string; def findsourceline(insertaddress, position, end = None, differences = None): nonlocal invertedmapranges nonlocal tlsfile, args, edited nonlocal foundfile, foundbase, foundoffset, foundlinenumber global object, tlsfilename if position != SourcePosition.diff \ and not (args.show_edit or args.do_edit): return (None, None) if tlsfile is None: if position == SourcePosition.diff: raise LogicError("Cannot show difference without a trace listing file!") if args.repeat: raise LogicError("Cannot edit without a trace listing file!") return (None, None) if (position != SourcePosition.diff or args.wlcalc_replace) and edited: print("Skipping subsequent edit!") return (None, None) patternsource = re.compile(r'^(.{40})(.*)$') patternlabel = \ re.compile(r"""(?xi)^\s*( (?:procedure\s+|entry\s+) [A-Za-z\._\$][A-Za-z\._\$0-9]*\b| [A-Za-z\._\$][A-Za-z\._\$0-9]*\b (?::|\s+proc\b(?:\s+far|\s+near)?|\s+label\b|\s+d[bwd]\b))""") foundlabels = {} patternoffset = re.compile(r'^\s{0,5}[0-9]{1,6} ([0-9A-Fa-f]{8}\b) \[?\(?[\?0-9A-Fa-f]{2}') patterndump = re.compile(r'^\s{0,5}[0-9]{1,6} [0-9A-Fa-f]{8}\b ([\]\[\)\(\?0-9A-Fa-f]{2,})') patternbrackets = re.compile(r'[\]\[\)\(]+') limit = 512 base = foundbase currentfile = None currentlinenumber = None firstrun = True strucs = [] dowlcalcreplace = 0 if foundoffset is not None and not args.only_second: tlsfile.seek(foundoffset, 0) currentfile = foundfile.copy() currentlinenumber = foundlinenumber - 1 else: currentfile = [None,] currentlinenumber = 0 base = 0 tlsfile.seek(0, 0) firstrun = False if object.debug & 257: if currentfile: print("findsourceline init last currentfile=%s" % currentfile[-1:][0]) repeat = True while repeat: repeat = False found = False founddiff = False offset = None lastlabel = None lastlabeldup = None linesafterlabel = 0 candidatefile = [None,] candidatelabel = None candidatelabeldup = None candidatelinesafterlabel = 0 matchsource = None lastlinehadoffset = False priorline = None priorlinesdeque = deque([]) savepriorline = True line = tlsfile.readline() while line: try: currentlinenumber += 1 if object.debug & 1: print(">%s<" % line.strip()) if firstrun and not limit and not founddiff: if object.debug & 257: print("Starting second run! Limit exceeded") firstrun = False repeat = True currentfile = [None,] currentlinenumber = 0 base = 0 tlsfile.seek(0, 0) break limit -= 1 match = re.search(r""" ^===\ Trace\ listing\ source:\ ([-+0-9A-Za-z\._\/\\:]+)\s*$ """, line, flags = re.X) if match is not None: foundlabels = {} currentfile = [match.group(1),] if object.debug & 257: print("Set source file findsourceline = %s" % currentfile[-1:][0]) match = re.search(r""" ^\s{0,5}[0-9]{1,6}\s{28,34}(?:<[0-9]+>\s*)?;===\ Push\ trace\ listing\ source: \ ([-+0-9A-Za-z\._\/\\:]+)\s*$ """, line, flags = re.X) if match is not None: foundlabels = {} currentfile.append(match.group(1),) if object.debug & 257: print("Push set source file findsourceline = %s" % currentfile[-1:][0]) match = re.search(r""" ^\s{0,5}[0-9]{1,6}\s{28,34}(?:<[0-9]+>\s*)?;===\ Pop\ trace\ listing\ source \s*$ """, line, flags = re.X) if match is not None: foundlabels = {} currentfile.pop() if object.debug & 257: print("Pop set source file findsourceline = %s" % currentfile[-1:][0]) match = re.search(r""" ^===\ Switch\ to\ base=(-?[0-9A-Fa-f]+)[Hh]?\s """, line, flags = re.X) if match is not None: base = int(match.group(1), 16) match = re.search(r"""(?xi) ^.{40}\s*([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*)\s+struc\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*struc\s+([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*) """, line) if match is not None: strucs.append(match.group(1)) if strucs: match = re.search(r"""(?xi) ^.{40}\s*""" + re.escape(strucs[-1]) + r"""\s+ends\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*endstruc\b """, line) if match is not None: strucs.pop() matchsource = re.search(patternsource, line) if matchsource is not None: if '******************' not in matchsource.group(1) \ and '------------------' not in matchsource.group(1): if object.debug & 513: print("Increment linesafterlabel, %u >%s<" % (linesafterlabel, line.strip())) linesafterlabel += 1 matchlabel = re.search(patternlabel, matchsource.group(2)) if matchlabel is not None: if object.debug & 513: print("Set lastlabel, >%s<" % line.strip()) lastlabel = matchlabel linesafterlabel = 0 labelstring = matchlabel.group(1) lastlabeldup = 0 if labelstring in foundlabels: lastlabeldup = foundlabels[labelstring] foundlabels[labelstring] += 1 else: foundlabels[labelstring] = 1 if object.debug & (2048 * 1024 + 1 + 512): print("Found label %s dup=%u" % \ (lastlabel.group(1), lastlabeldup)) if lastlinehadoffset: candidatefile = currentfile.copy() candidatelabel = lastlabel candidatelabeldup = lastlabeldup candidatelinesafterlabel = linesafterlabel candidatecheckline = priorline if object.debug & 513: print("Set clal=%u cl=%s >%s<" \ % (candidatelinesafterlabel, candidatelabel, line.strip())) lastlinehadoffset = False matchoffset = re.search(patternoffset, line) if not strucs and matchoffset is not None: if position == SourcePosition.afterprior: lastlinehadoffset = True if object.debug & 1: print("Set had offset >%s<" % (line.strip())) offset = int(matchoffset.group(1), 16) + base + int(args.offset, 0) if object.debug & 1: print("Found offset %06X" % offset) dumplength = 1 matchdump = re.search(patterndump, line) if matchdump is not None: dumpstripped = re.sub(patternbrackets, "", matchdump.group(1)) dumplength = len(dumpstripped) // 2 if object.debug & 3: print("Found offset %06X length %X" % (offset, dumplength)) if position == SourcePosition.diff: # not (b <= c or a >= d) # a = insertaddress, b = end, # c = offset, d = offset + dumplength if (object.debug & (32768 * 1024 + 1)): print("ins=%06Xh end=%06Xh ofs=%06Xh len=%04Xh" % ( insertaddress, end, offset, dumplength)) if not (end <= offset or insertaddress >= offset + dumplength): if not founddiff: if currentfile[-1] != None: print("=== File: %s" % currentfile[-1], end='') if lastlabel is not None: label = lastlabel.group(1) counter = lastlabeldup + 1 suffix = "th" if (counter != 11 and (counter % 10) == 1): suffix = "st" if (counter != 12 and (counter % 10) == 2): suffix = "nd" if (counter != 13 and (counter % 10) == 3): suffix = "rd" if counter > 1: print(", %u%s label %s" % ( counter, suffix, label)) else: print(", label %s" % label) else: print(", no label") founddiff = True liner = line liner = re.sub(r'[\r\n]+', '', liner) liner = re.sub(r'^(.{25,35} ) {18}', r'\1', liner) liner = re.sub(r'^(.{25,35} ) {14}', r'\1', liner) liner = re.sub(r'^([^;]+?)\s\s+;', r'\1\t;', liner) # liner = re.sub(r'\s+', ' ', liner) liner = re.sub(r'^\s*[0-9]+\s+', '', liner, count=1) liner = re.sub(r'^0000', '', liner, count=1) linerplain = liner wlcalcliner = re.sub(r"""(?xi) ^[0-9A-Fa-f]{4,8}\s+ ([\]\[\)\(\?0-9A-Fa-f]+) .*$ """, "\\1", liner, count = 1) firstwlcalc = 1 for adjustment in range(0, dumplength): if offset + adjustment in differences: if args.wlcalc_replace: # Mark wlcalc different bytes with <...> wlcalcliner = re.sub(r"""(?xi) ^ ((?:[][)(]* (?:%s)? [\?0-9A-Fa-f]{2} (?:%s)? [][)(]* ){%u}) ([][)(]*) ([\?0-9A-Fa-f]{2})""" % ("<", ">", adjustment), "\\1\\2<\\3>", wlcalcliner, count = 1) if args.show_diff and args.show_diff_colour: # Mark -J different bytes with ESC[7m...ESC[m liner = re.sub(r"""(?xi) ^([0-9A-Fa-f]{4,8}\s+) ((?:[][)(]* (?:%s)? [\?0-9A-Fa-f]{2} (?:%s)? [][)(]* ){%u}) ([][)(]*) ([\?0-9A-Fa-f]{2})""" % ("\x1B\\[7m", "\x1B\\[m", adjustment), "\\1\\2\\3\x1B[7m\\4\x1B[m", liner, count = 1) # Remove reset escapes immediately followed by highlight again liner = re.sub("\x1B\\[m\x1B\\[7m", '', liner) if args.wlcalc_replace: # Remove wlcalc bytes already marked relocated wlcalcliner = re.sub(r"""(?xi) [\[\(] [<>\?0-9A-Fa-f]+ [\]\)] """, " ", wlcalcliner); # Only count if at least two continous bytes # remaining in the machine code matchwlcalc = re.search(r"><", wlcalcliner) if matchwlcalc is not None: dowlcalcreplace += firstwlcalc firstwlcalc = 0 if args.show_diff: beforedeque = deque([]) dequecounter = 1 while re.search(r"""(?xi) ^[0-9A-Fa-f]+\s+[0-9A-Fa-f]+$ """, linerplain) \ and len(priorlinesdeque) >= dequecounter: linerplain = priorlinesdeque[- dequecounter] linerplain = re.sub(r'[\r\n]+', '', linerplain) linerplain = re.sub(r'^(.{25,35} ) {18}', r'\1', linerplain) linerplain = re.sub(r'^(.{25,35} ) {14}', r'\1', linerplain) linerplain = re.sub(r'^([^;]+?)\s\s+;', r'\1\t;', linerplain) # linerplain = re.sub(r'\s+', ' ', linerplain) linerplain = re.sub(r'^\s*[0-9]+\s+', '', linerplain, count=1) linerplain = re.sub(r'^0000', '', linerplain, count=1) beforedeque.appendleft( \ "%u: %s" % \ (currentlinenumber - dequecounter, \ linerplain)) dequecounter += 1 for dequeline in beforedeque: print(dequeline) print("%u: %s" % (currentlinenumber, liner)) priorlinesdeque.clear() savepriorline = False if dowlcalcreplace == 1: markearliest("%06X " % offset, offset, 1) candidatelabel = lastlabel candidatelabeldup = lastlabeldup candidatelinesafterlabel = linesafterlabel candidatecheckline = matchsource.group(2) sourcefile = currentfile[-1:][0] found = True if not args.show_diff: break elif insertaddress >= offset and insertaddress < offset + dumplength: if position == SourcePosition.exactly: candidatefile = currentfile.copy() candidatelabel = lastlabel candidatelabeldup = lastlabeldup candidatelinesafterlabel = linesafterlabel candidatecheckline = matchsource.group(2) if object.debug & 1: print("Set exactly clal=%u cl=%s >%s<" \ % (candidatelinesafterlabel, candidatelabel, line.strip())) if candidatelabel is not None: sourcefile = candidatefile[-1:][0] found = True break else: if not firstrun: raise ListingError("No label found in second run") if object.debug & 1: print("Starting second run!") firstrun = False repeat = True currentfile = [None,] base = 0 tlsfile.seek(0, 0) break if (args.show_diff): if savepriorline: priorlinesdeque.append(line) if (len(priorlinesdeque) > 16): priorlinesdeque.popleft() savepriorline = True finally: if (not found or args.wlcalc_replace) and not repeat: if matchsource is not None: priorline = matchsource.group(2) line = tlsfile.readline() print("Searching insert address %06X" % insertaddress) if position == SourcePosition.diff: if not founddiff: idx = 0 while idx < len(invertedmapranges): startrange = invertedmapranges[idx][0] endrange = invertedmapranges[idx][1] # not (b <= c or a >= d) # a = insertaddress, b = end, # c = startrange, d = endrange + 1 if not (end < startrange or insertaddress > endrange): print("Differrence is in inverted .map range " + \ "%06Xh to %06Xh" % (startrange, endrange)) return (None, None) idx += 1 raise ListingError("Listing text line not found!") if (not found or not args.wlcalc_replace): return (None, None) if not found: if args.repeat: if object.debug & 257: print("Not found before source scan") raise ListingError("Source text line not found!") return (None, None) (label, labeldup, linesafterlabel) = \ (candidatelabel, candidatelabeldup, candidatelinesafterlabel) checkline = expandtabs(candidatecheckline).strip() if object.debug & (1 + 65536 * 1024): print("Check line >%s<" % checkline.strip()) if sourcefile is None: sourcefile = tlsfilename if labeldup: print("Search file \"%s\" after %u duplicates label \"%s\" +%u" \ % (sourcefile, labeldup, label.group(1), linesafterlabel)) else: print("Search file \"%s\" label \"%s\" +%u" \ % (sourcefile, label.group(1), linesafterlabel)) if args.pattern == None: filename = re.sub(r'\.lst$', '.nas', sourcefile) else: filename = sourcefile for combinedpattern in args.pattern: splitpatterns = combinedpattern.split("::") if not (len(splitpatterns) >= 1 and len(splitpatterns) <= 3): raise InputError("Invalid -p pattern in \"%s\"" % combinedpattern) search = splitpatterns[0] replace = '' if len(splitpatterns) >= 2: replace = splitpatterns[1] count = 1 if len(splitpatterns) == 3: count = int(splitpatterns[2], 0) if object.debug & (1 + 65536 * 1024): print("Sub search>%s< replace>%s< filename>%s<" \ % (search, replace, filename)) if filename is None and search == '': filename = '' if filename is None and search == '': filename = '' search = '' filename = re.sub(search, replace, filename, count = count) if object.debug & 1: print("Sub done>%s<" \ % (filename)) if not os.path.isfile(filename): if args.include: for dir in args.include: if os.path.isfile(dir + filename): filename = dir + filename break if not os.path.isfile(filename): print("Error: File \"%s\" to edit not found in include paths!" % filename) with codecs.open(filename, mode='r', encoding='latin-1') as file: state = 0 lines = () linenumberlist = 0 linenumbersource = 1 prependline = '' listskipped = 0 for line in file: listskipped += 1 if args.nasm and re.search(r'\\[\r\n]*$', line): prependline += re.sub(r'\\[\r\n]*$', '', line, count = 1) continue line = prependline + line prependline = '' linenumberlist += 1 lines += ([line, linenumberlist, linenumbersource],) if len(lines) > 8 and not object.debug & 33: lines = lines[1:] if state == 0: matchsourcelabel = re.search(patternlabel, line) if matchsourcelabel is not None \ and expandtabs(matchsourcelabel.group(1)) \ == expandtabs(label.group(1)): if labeldup: labeldup -= 1 else: state = 1 if state == 1: targetlinelist = linesafterlabel + linenumberlist # print("st1 lst%u/src%u +%u target=%u skip%u\n" % (linenumberlist, linenumbersource, linesafterlabel, targetlinelist, listskipped)) state = 2 if state == 2: # print("st2 lst%u/src%u +%u target=%u skip%u\n" % (linenumberlist, linenumbersource, linesafterlabel, targetlinelist, listskipped)) if targetlinelist == linenumberlist: state = 3 if state == 3: matches = False def comparecheckline(sourceline, listline): if re.search(r'\\\s*[\r\n]*$', sourceline): sourceline = re.sub(r'\\\s*[\r\n]*$', '', sourceline) listline = re.sub(r'^(.{%u}).*$' % len(sourceline), r'\1', listline, count = 1) return sourceline == listline print("Found at file \"%s\" line lst%u/src%u" % (filename, linenumberlist, linenumbersource)) if position == SourcePosition.afterprior: secondtolastline = expandtabs(lines[-2:][0][0]).strip() matches = comparecheckline(secondtolastline, checkline) if object.debug & 1 or not matches: print("2nd>%s< check>%s<" % (secondtolastline, checkline)) elif position == SourcePosition.exactly \ or position == SourcePosition.diff and args.wlcalc_replace: lastline = expandtabs(lines[-1:][0][0]).strip() matches = comparecheckline(lastline, checkline) if object.debug & 1 or not matches: print("last>%s< check>%s<" % (lastline, checkline)) else: assert False, "Unknown source position passed down" if args.show_edit: for tuple in lines: (liner, numberlist, numbersource) = tuple liner = re.sub(r'[\r\n]+', '', liner) print("lst%u/src%u: %s" % (numberlist, numbersource, liner)) if not matches: raise ListingError("Check line doesn't match") return (filename, linenumbersource) linenumbersource += listskipped listskipped = 0 if args.repeat: if object.debug & 257: print("Not found after source scan") raise ListingError("Source text line not found!") return (None, None) def matchsize(string): string = re.sub(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*(?:at)\s+[^,;]+,\s* """, '', string, count = 1) string = re.sub(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*(?:fill|fill_at_least)\s+[^,;]+,[^,;]+,\s* """, '', string, count = 1) matchdata = re.search(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*d([bwd])\b """, string) matchaltdatabyte = re.search(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*(\.string|\.ascii|\.byte|\.skip|\.align| \.p2align|\.balign|counted|ascii|asciz|align)\b """, string) matchaltdataword = re.search(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*(critpatchentry|short_addr|write_entrypoint_list| \$m_id|\.hword|repeatdw|immasm_special_dump|isvariablestruc| areadefine|installflag)\b """, string) matchaltdataiam = re.search(r"""(?xi) ^\s*(?:[A-Za-z_\.\$][A-Za-z_\.\$0-9]*(?::|\s))? \s*I_am\s+[^,;]+\s*,\s*([^,;]+)\b """, string) if matchdata is not None: return matchdata.group(1).lower() elif matchaltdatabyte is not None: return "b" elif matchaltdataword is not None: return "w" elif matchaltdataiam is not None: if matchaltdataiam.group(1).lower == "dword": return "d" elif matchaltdataiam.group(1).lower == "word": return "w" else: return "b" return None def handlerange(offset): nonlocal lastdisassembly, priorarray, array nonlocal f1, f2, buffer1, buffer2, idx, headersize nonlocal relocs1, relocs2 nonlocal tlsfile nonlocal args nonlocal amountnodifferenceranges, amountdifferencelines nonlocal earliestdifferenceflag nonlocal maximum nonlocal foundfile, foundbase, foundoffset, foundlinenumber nonlocal invertedmapranges global object fullarray = priorarray + array address = -1 for bit in fullarray: address += 1 if (address < lastdisassembly): continue if (bit): start = max(0, address - 16) break assert bit, "Unexpected array content" end = min(offset + idx, int(args.maximum_offset, 0) + 1 + 16) firstdifference = address lastdifference = address address -= 1 for bit in fullarray[address + 1:]: address += 1 if (bit): lastdifference = address if (address >= end): break found = 0 tlsoffset = None foundoffset = None foundlinenumber = None foundfile = [None,] currentfile = [None,] currentlinenumber = 0 strucs = [] priortlssize = 'u' currenttlssize = 'u' if (object.debug & (32768 * 1024 + 1)): print("before start=%06Xh" % (start)) if tlsfile is not None: tlsfile.seek(0, 0) base = 0 currentlineoffset = 0 line = tlsfile.readline() lastlinesize = None while line: try: currentlinenumber += 1 match = re.search(r""" ^===\ Trace\ listing\ source:\ ([-+0-9A-Za-z\._\/\\:]+)\s*$ """, line, flags = re.X) if match is not None: lastlinesize = None currentfile = [match.group(1),] if object.debug & 1: print("Set source file handlerange = %s" % currentfile[-1:][0]) match = re.search(r""" ^\s{0,5}[0-9]{1,6}\s{28,34}(?:<[0-9]+>\s*)?;===\ Push\ trace\ listing\ source: \ ([-+0-9A-Za-z\._\/\\:]+)\s*$ """, line, flags = re.X) if match is not None: lastlinesize = None currentfile.append(match.group(1),) if object.debug & 1: print("Push set source file handlerange = %s" % currentfile[-1:][0]) match = re.search(r""" ^\s{0,5}[0-9]{1,6}\s{28,34}(?:<[0-9]+>\s*)?;===\ Pop\ trace\ listing\ source \s*$ """, line, flags = re.X) if match is not None: lastlinesize = None currentfile.pop() if object.debug & 1: print("Pop set source file handlerange = %s" % currentfile[-1:][0]) match = re.search(r""" ^===\ Switch\ to\ base=(-?[0-9A-Fa-f]+)[Hh]?\s """, line, flags = re.X) if match is not None: lastlinesize = None base = int(match.group(1), 16) match = re.search(r"""(?xi) ^.{40}\s*([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*)\s+struc\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*struc\s+([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*) """, line) if match is not None: strucs.append(match.group(1)) if strucs: match = re.search(r"""(?xi) ^.{40}\s*""" + re.escape(strucs[-1]) + r"""\s+ends\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*endstruc\b """, line) if match is not None: strucs.pop() match = re.search(r""" ^\ {0,5}[0-9]{1,6}\ ([0-9A-Fa-f]{4,8})\s+ ([\]\[\)\(\?0-9A-Fa-f]+)-?(\s+ .*)?$ """, line, flags = re.X) if not strucs and match is not None: matchedsize = matchsize(match.group(3)) if matchedsize is not None: currenttlssize = matchedsize lastlinesize = currenttlssize if object.debug & (4096 * 1024): liner = re.sub(r'^\s+|\s+$|[\r\n]+', '', line) print("size=%s liner=>%s<" % (currenttlssize, liner)) else: matchother = re.search(r"""(?xi) ^\s*[^\s;]+ """, match.group(3)) if matchother is not None: currenttlssize = "u" lastlinesize = currenttlssize elif lastlinesize is not None: currenttlssize = lastlinesize match = re.search(r""" ^(\ {0,5}[0-9]{1,6}\ ([0-9A-Fa-f]{4,8})\s+ ([\]\[\)\(\?0-9A-Fa-f]+)-?(?:\s|$)) """, line, flags = re.X) if strucs: continue if match is None: continue if len(match.group(1)) > 40: continue dump = re.sub(r'[\]\[\)\(]+', '', match.group(3)) amount = len(dump) // 2 offset_line = base + int(match.group(2), 16) if object.debug & 16384: print("%06Xh amount=%u match=>%s<" % (offset_line, amount, match.group(1).strip())) if not amount: continue if firstdifference - int(args.offset, 0) >= offset_line + amount: continue if offset_line > firstdifference - int(args.offset, 0): continue if object.debug & 65: print("line=>%s< offset_line=%u dump=>%s< amount=%u argsoffset=%u" % (line, offset_line, dump, amount, int(args.offset, 0))) start = offset_line + int(args.offset, 0) found = 1 foundline = line foundbase = base foundoffset = currentlineoffset foundlinenumber = currentlinenumber foundfile = currentfile.copy() if object.debug & 65: print("handlerange set last foundfile=%s" % foundfile[-1:][0]) priortlssize = currenttlssize if start == firstdifference: break finally: currentlineoffset = tlsfile.tell() line = tlsfile.readline() if found and object.debug & 65: foundline = re.sub(r'\s{2,}', ' ', foundline) foundline = re.sub(r'\s+$', ' ', foundline) print("offset=%u size='%s' number=%u line=>%s< base=%06Xh" \ % (foundoffset, priortlssize, foundlinenumber, foundline, foundbase)) if (object.debug & (32768 * 1024 + 1)): print("after start=%06Xh" % (start)) ranges = invertedmapranges if not found: if (object.debug & (16384 * 1024 + 1)): print("notfound first=%06Xh=%u last=%06Xh=%u" % \ (firstdifference, firstdifference, lastdifference, lastdifference)) idx = 0 while idx < len(ranges): if (object.debug & (16384 * 1024 + 1)): print(ranges) if firstdifference >= ranges[idx][0] \ and firstdifference <= ranges[idx][1]: start = firstdifference break idx += 1 auto = int(args.auto_length, 0) autodisplay = int(args.auto_display, 0) if auto and (end - start) > auto: if not autodisplay: autodisplay = auto end = min(end, start + autodisplay + 16) lastdifference = min(lastdifference + 1, end) - 1 end = lastdifference + 16 args.maximum_offset = ("0x%06X" % end) print("Automatic difference length exceeded!") print("%06X up to below %06X, first=%06X last=%06X" % (start, end, firstdifference, lastdifference)) lastdisassembly = end if end <= headersize: return disbuffer1 = b'' disbuffer2 = b'' if start >= offset: disbuffer1 += buffer1[start - offset : end - offset] disbuffer2 += buffer2[start - offset : end - offset] else: f1original = f1.tell() minus = 0 if start > 0: minus = 1 f1.seek(start - minus, 0) disbuffer1 = bytearray(f1.read(end - (start - minus))) f1.seek(f1original, 0) f2original = f2.tell() f2.seek(start - minus, 0) disbuffer2 = bytearray(f2.read(end - (start - minus))) f2.seek(f2original, 0) length = min(len(disbuffer1), len(disbuffer2)) applyrelocations(disbuffer1, start - minus - headersize, length, relocs1) applyrelocations(disbuffer2, start - minus - headersize, length, relocs2) if minus: del disbuffer1[0] del disbuffer2[0] tlsdirectives = () done = 0 if object.debug & (512 * 1024 + 1): print("###", end="") print(ranges); if tlsfile is not None and foundoffset is not None: if object.debug & 4097: print("foundoffset=%06Xh" % (foundoffset)) tlsfile.seek(foundoffset, 0) base = foundbase strucs = [] line = tlsfile.readline() while line: try: match = re.search(r"""(?xi) ^===\ Switch\ to\ base=(-?[0-9A-Fa-f]+)[Hh]?\s """, line, flags = re.X) if match is not None: base = int(match.group(1), 16) match = re.search(r"""(?xi) ^.{40}\s*([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*)\s+struc\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*struc\s+([A-Za-z\$\@\_\.][0-9A-Za-z\$\@\_\.]*) """, line) if match is not None: strucs.append(match.group(1)) if strucs: match = re.search(r"""(?xi) ^.{40}\s*""" + re.escape(strucs[-1]) + r"""\s+ends\b """, line) if match is None: match = re.search(r"""(?xi) ^.{40}\s*endstruc\b """, line) if match is not None: strucs.pop() match = re.search(r"""(?xi) ^(\ {0,5}[0-9]{1,6}\ ([0-9A-Fa-f]{4,8})\s+ ([\]\[\)\(\?0-9A-Fa-f]+)-?)(\s+ .*)?$ """, line, flags = re.X) if strucs: continue if match is None: continue if len(match.group(1)) > 40: continue dump = re.sub(r'[\]\[\)\(]+', '', match.group(3)) amount = len(dump) // 2 offset_line = base + int(match.group(2), 16) directivestart = offset_line + int(args.offset, 0) # if directivestart > lastdifference + 16 + 1: # break if re.search(r"^\s*$", match.group(4)): if object.debug & 8192: print("%06Xh amount=%u priortlssize='%s' match=>%s<" % (directivestart, amount, priortlssize, match.group(1).strip())) tlsdirectives += \ ([directivestart, amount, priortlssize],) else: matchedsize = matchsize(match.group(4)) if object.debug & 1024: print("%06Xh matchdata=%s matchgroup=%s" % (directivestart, matchdata, match.group(4).strip())) if matchedsize is not None: priortlssize = matchedsize if object.debug & (4096 * 1024): liner = re.sub(r'^\s+|\s+$|[\r\n]+', '', line) print("size=%s liner=>%s<" % (priortlssize, liner)) if object.debug & 2048 and priortlssize == "b": print("%06Xh matchdata=%s matchgroup=%s" % (directivestart, matchdata, match.group(4).strip())) if object.debug & (1024 * 1024): print("@@@1", end="") print(tlsdirectives) tlsdirectives += \ ([directivestart, amount, priortlssize],) if object.debug & (1024 * 1024): print("@@@2", end="") print(tlsdirectives) else: matchother = re.search(r"""(?xi) ^\s*[^\s;]+ """, match.group(4)) if matchother is not None: priortlssize = "u" if object.debug & (1024 * 1024): print("@@@", end="") print(ranges) if amount: ranges += ([directivestart, directivestart + amount - 1],) ranges = sorted(ranges, key=lambda tuple: tuple[0]) mergedranges = () idx = 0 while idx < len(ranges): startrange = ranges[idx][0] endrange = ranges[idx][1] if object.debug & (1024 * 1024): print("outer startrange=%06Xh=%u endrange=%06Xh=%u idx=%u" % (startrange, startrange, endrange, endrange, idx)) while idx + 1 < len(ranges) \ and endrange + 1 >= ranges[idx + 1][0]: endrange = max(endrange, ranges[idx + 1][1]) idx += 1 if object.debug & (1024 * 1024): print("inner startrange=%06Xh=%u endrange=%06Xh=%u idx=%u" % (startrange, startrange, endrange, endrange, idx)) mergedranges += ([startrange, endrange],) idx += 1 ranges = mergedranges if object.debug & (128*1024): for tuple in ranges: print(">>%06X up to below %06X" % (tuple[0], tuple[1] + 1)) idx = 0 while idx < len(ranges): if firstdifference >= ranges[idx][0] \ and lastdifference + 16 + 1 <= ranges[idx][1]: if object.debug & (256 * 1024 + 1): print(("firstdiff=%u lastdiffplus=%u" + " rangesidx0=%u rangesidx1=%u") % (firstdifference, lastdifference, ranges[idx][0], ranges[idx][1])); done = 1 break idx += 1 if done: break finally: line = tlsfile.readline() if not done and False: # commented out, seems not needed # this is handled by addlistu if (object.debug & (16384 * 1024 + 1)): print("first=%06Xh=%u last=%06Xh=%u" % \ (firstdifference, firstdifference, lastdifference, lastdifference)) idx = 0 while idx < len(ranges): if (object.debug & (16384 * 1024 + 1)): print(ranges) if firstdifference >= ranges[idx][0] \ and lastdifference <= ranges[idx][1]: if object.debug & (256 * 1024 + 1): print(("firstdiff=%u lastdiffplus=%u" + " rangesidx0=%u rangesidx1=%u") % (firstdifference, lastdifference, ranges[idx][0], ranges[idx][1])); done = 1 tlsdirectives += ([ranges[idx][0], \ ranges[idx][1] - ranges[idx][0] + 1, 'b'],) if (object.debug & (16384 * 1024 + 1)): print(tlsdirectives) break idx += 1 tlsdirectives = sorted(tlsdirectives, key=lambda tuple: tuple[0]) if object.debug & (128*1024): print(tlsdirectives) for tuple in tlsdirectives: if tuple[0] >= 0x8110 and tuple[0] <= 0x8130: print("tls directive tuple=", end="") print(tuple) pass pattern = re.compile(r'^([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) (.*)') badpattern = re.compile(r'^[0-9A-Fa-f:]{9,13}\s+-?\s+XX') repeated = 0 repeat = 1 while repeat: repeat = 0 text1 = disassemble(disbuffer1, start, \ lastdifference + 1 - start, tlsdirectives) lines1 = text1.split('\n') index1 = 0 while index1 < len(lines1): match1 = pattern.search(lines1[index1]) if match1 is None: repeated += 1 if not repeated <= 16: raise LogicError("Repetition limit reached") repeat = 1 print("Repeating due to malformed line 1 \"%s\"\x07" % lines1[index1]) assert not re.search(badpattern, lines1[index1]), \ "Bad line pattern matched, not repeating" break index1 += 1 repeated = 0 repeat = 1 while repeat: repeat = 0 text2 = disassemble(disbuffer2, start, \ lastdifference + 1 - start, tlsdirectives) lines2 = text2.split('\n') index2 = 0 while index2 < len(lines2): match2 = pattern.search(lines2[index2]) if match2 is None: repeated += 1 if not repeated <= 16: raise LogicError("Repetition limit reached") repeat = 1 print("Repeating due to malformed line 2 \"%s\"\x07" % lines2[index2]) assert not re.search(badpattern, lines2[index2]), \ "Bad line pattern matched, not repeating" break index2 += 1 index1 = index2 = 0 while not args.dump_all and index1 < len(lines1) and index2 < len(lines2): lines2[index2] = swapoperands(lines1[index1], lines2[index2]) (addindex1, addindex2, proceed, isfuzzy) = \ handlenops(lines1, index1, lines2, index2) if isfuzzy or not proceed: break index1 += addindex1 + 1 index2 += addindex2 + 1 if index1 >= len(lines1) and index2 >= len(lines2): print("no difference") amountnodifferenceranges += 1 differences = [] if args.show_dump_colour: position = start for byte1, byte2 in zip(disbuffer1, disbuffer2): if byte1 != byte2: differences.append(position) if object.debug & 32769: print("%06Xh" % position) position += 1 padlength = 0 if args.side_by_side: while index1 < len(lines1) and index2 < len(lines2): lines2[index2] = swapoperands(lines1[index1], lines2[index2]) (addindex1, addindex2, proceed, isfuzzy) = \ handlenops(lines1, index1, lines2, index2) line1 = lines1[index1] firstline = ("first: %s" % line1) line2 = lines2[index2] if proceed and (addindex1 or addindex2): line2replaced = line2 line2replaced += " ; sizediff" if isfuzzy is not None and isfuzzy: line2replaced += " ; fuzzysame" line2replaced = highlightdump(line2replaced, differences) firstline = ("first: %s" % line1) secondline = ("second: %s" % line2replaced) padlength = 40 while padlength - len(firstline) <= 2: padlength += 8 buf = "%-*s%s" % (padlength, firstline, secondline) buf = highlightdump(buf, differences) print(buf) index1 += 1 index2 += 1 while addindex1: line1 = lines1[index1] line1 = highlightdump(line1, differences) firstline = ("first: %s ; sizediff" % line1) print(firstline) index1 += 1 addindex1 -= 1 while addindex2: line2 = lines2[index2] line2 = highlightdump(line2, differences) print("%-*ssecond: %s ; sizediff" % (40, "", line2)) index2 += 1 addindex2 -= 1 continue pattern = re.compile(r'^([0-9A-Fa-f]{6}) ') match1 = pattern.search(line1) match2 = pattern.search(line2) if (match1 is None or match2 is None): print("Error: 1>%s< 2>%s<" % (line1, line2)) assert False hex1 = int(match1.group(1), 16) hex2 = int(match2.group(1), 16) if maximum is not None and hex1 > maximum and hex2 > maximum: print("Difference length after earliest difference reached!") return line2replaced = line2 if line1 == line2: line2replaced = "samesame" elif (hex1 == hex2): line2replaced = re.sub(pattern, 'same ', line2, count=1) else: assert hex1 != hex2, "Expected different addresses" if hex2 > hex1: target = index1 + 1 targetfound = 0 while target < len(lines1): match3 = pattern.search(lines1[target]) if (match3 is None): print("Error: 3>%s<" % (lines1[target])) assert False hex3 = int(match3.group(1), 16) if hex3 == hex2: targetfound = 1 while index1 < target: line1 = lines1[index1] markearliest(line1, start, 0) line1 = highlightdump(line1, differences) firstline = ("first: %s" % line1) print(firstline) amountdifferencelines += 1 index1 += 1 break elif hex3 > hex2: break target += 1 if targetfound: continue else: target = index2 + 1 targetfound = 0 while target < len(lines2): match3 = pattern.search(lines2[target]) if (match3 is None): print("Error: 3>%s<" % (lines2[target])) assert False hex3 = int(match3.group(1), 16) if hex3 == hex1: targetfound = 1 while index2 < target: line2 = lines2[index2] markearliest(line2, start, 0) line2 = highlightdump(line2, differences) print("%-*ssecond: %s" % (40, "", line2)) amountdifferencelines += 1 index2 += 1 break elif hex3 > hex1: break target += 1 if targetfound: continue if "samesame" not in line2replaced: if fuzzycompare(line1, line2): line2replaced += " ; fuzzysame" else: if not checksegmentoverride(line1, line2): if not checkfardd(line1, line2): pass patternnop = re.compile( r'^[0-9A-Fa-f]{6} \+[0-9A-Fa-f]+ nop$') matchnop1 = re.search(patternnop, line1) matchnop2 = re.search(patternnop, line2) if matchnop1 is None and matchnop2 is not None \ or matchnop1 is not None and matchnop2 is None: nop = 1 if matchnop1 is None: nop = 2 if args.edit_file == 0 or args.edit_file == nop ^ 3: print("hint: insert a NOP in file %u" % (nop ^ 3)) match = re.search(r'^([0-9A-F]{6}) ', line2) insertaddress = int(match.group(1), 16) editsource(findsourceline(insertaddress, SourcePosition.afterprior), SourceEdit.insertline, insert="nop") else: print("hint: remove a NOP in file %u" % (nop)) line2replaced = highlightdump(line2replaced, differences) firstline = ("first: %s" % line1) secondline = ("second: %s" % line2replaced) if not re.search(r'same$', line2replaced): markearliest(line1, start, 0) padlength = 40 while padlength - len(firstline) <= 2: padlength += 8 buf = "%-*s%s" % (padlength, firstline, secondline) buf = highlightdump(buf, differences) print(buf) if "samesame" not in line2replaced: amountdifferencelines += 2 index1 += 1 index2 += 1 padlength = 40 while index1 < len(lines1): pattern = re.compile(r'^([0-9A-Fa-f]{6}) ') match1 = pattern.search(lines1[index1]) if (match1 is None): print("Error: 1>%s<" % lines1[index1]) assert False hex1 = int(match1.group(1), 16) if maximum is not None and hex1 > maximum: print("Difference length after earliest difference reached!") break line = highlightdump(lines1[index1], differences) print("first:\t%s" % line) amountdifferencelines += 1 index1 += 1 while index2 < len(lines2): pattern = re.compile(r'^([0-9A-Fa-f]{6}) ') match2 = pattern.search(lines2[index2]) if (match2 is None): print("Error: 2>%s<" % lines2[index2]) assert False hex2 = int(match2.group(1), 16) if maximum is not None and hex2 > maximum: print("Difference length after earliest difference reached!") break line = highlightdump(lines2[index2], differences) print("%-*ssecond:\t%s" % (padlength, "", line)) amountdifferencelines += 1 index2 += 1 if args.show_diff or args.wlcalc_replace: pattern = re.compile(r'^([0-9A-Fa-f]{6}) \+([0-9A-Fa-f]+) ') if args.edit_file == 1: matchstart = pattern.search(lines1[0]) matchend = pattern.search(lines1[-1]) if matchstart is None: print("Error: start>%s<" % lines1[0]) assert False if matchend is None: print("Error: end>%s<" % lines1[-1]) assert False else: matchstart = pattern.search(lines2[0]) matchend = pattern.search(lines2[-1]) if matchstart is None: print("Error: start>%s<" % lines2[0]) assert False if matchend is None: print("Error: end>%s<" % lines2[-1]) assert False start = int(matchstart.group(1), 16) end = int(matchend.group(1), 16) + int(matchend.group(2), 16) if object.debug & 1: print("start=%06Xh end=%06Xh" % (start, end)) differences = [] if args.show_diff_colour or args.wlcalc_replace: position = start for byte1, byte2 in zip(disbuffer1, disbuffer2): if byte1 != byte2: differences.append(position) position += 1 rettuple = findsourceline(start, SourcePosition.diff, \ end = end, differences = differences) if args.wlcalc_replace and rettuple[0] is not None: editsource(rettuple, SourceEdit.wlcalc) def pruef(file, name = None, suminput = 1): file.seek(0, 0) sum = suminput while byte := file.read(1): sum *= 31 sum += byte[0] sum &= 0xFFFF file.seek(0, 0) if name is not None: print(name + " checksum is %04Xh" % sum) return sum def applyrelocations(buffer, offsetstart, length, relocs, carry = None): nonlocal applyreloc if not applyreloc: return 0 offsetend = offsetstart + length if offsetend <= 0: return 0 if carry is None: carry = 0 for key in sorted(relocs): if offsetstart <= key < offsetend: value = buffer[key - offsetstart] result = (value + applyreloc) & 255 carry = result < value buffer[key - offsetstart] = result if offsetstart <= (key + 1) < offsetend: value = buffer[key - offsetstart + 1] result = (value + (applyreloc >> 8) + carry) & 255 carry = 0 buffer[key - offsetstart + 1] = result return carry global object, objectinitialised, globalrepeat, repeating global priorargs global priorearliestdifferenceflag global tlsfilename buffersize = 8192 parser = argparse.ArgumentParser( prog='ident86', description='Compares 8086 executables.') parser.add_argument('filenames', nargs='*') parser.add_argument('-m', '--minimum-offset', default='0') parser.add_argument('-M', '--maximum-offset', default='0xFFFFffff') parser.add_argument('-z', '--skip-header', action='store_true') parser.add_argument('-Z', '--apply-reloc', default='0') parser.add_argument('-x', '--only-second', action='store_true') parser.add_argument('-a', '--auto-length', default='0') parser.add_argument('-A', '--auto-display', default='0') parser.add_argument('-s', '--side-by-side', action='store_true') parser.add_argument('-o', '--offset', default='+0') parser.add_argument('-v', '--version', action='store_true') parser.add_argument('-V', '--no-version', action='store_true') parser.add_argument('-d', '--difference-length', default='0') parser.add_argument('-X', '--trailer-size', default='16') parser.add_argument('-e', '--edit-file', default='0', type=int, choices=range(0,3)) parser.add_argument('-E', '--do-edit', action='store_true') parser.add_argument('-S', '--show-edit', action='store_true') parser.add_argument('-j', '--show-diff', action='store_true') parser.add_argument('-J', '--show-diff-colour', action='store_true') parser.add_argument('-c', '--cookie', default='') parser.add_argument('-b', '--build', default='') parser.add_argument('-p', '--pattern', action='append', default=None) parser.add_argument('-I', '--include', action='append', default=None) parser.add_argument('-P', '--pruef', action='store_true') parser.add_argument('-t', '--time', action='store_true') parser.add_argument('-T', '--time-utc', action='store_true') parser.add_argument('-r', '--repeat', action='store_true') parser.add_argument('-f', '--fuzzy', default='32') parser.add_argument('-D', '--dump-all', action='store_true') parser.add_argument('-Y', '--show-dump-colour', action='store_true') parser.add_argument('-F', '--forced', action='append', default=None) parser.add_argument('-w', '--wlcalc-replace', action='store_true') parser.add_argument('-n', '--nasm', action='store_true') parser.add_argument('-W', '--wlcalc-quick', action='store_true') if not repeating: priorargs = args = parser.parse_args() else: args = priorargs if args.version: # disable all further actions and checks args.no_version = False args.build = '' args.cookie = '' args.maximum_offset = "0" args.minimum_offset = "0" args.apply_reloc = "0" args.do_edit = False args.show_edit = False args.show_diff = False args.repeat = False if int(args.apply_reloc, 0) < 0 or int(args.apply_reloc, 0) > 0xFFFF: raise LogicError("Switch -Z invalid number") if (args.do_edit or args.show_edit) and args.edit_file == 0: raise LogicError("Using switches -S or -E requires switch -e") if not args.do_edit and args.repeat: raise LogicError("Using switch -r requires switch -E") if args.build != '': rc = subprocess.call(args.build, shell = True) if rc: raise InputError("Build scriptlet failed!") if args.time_utc: print(datetime.now(timezone.utc).strftime("=== %Y-%m-%d %H:%M:%S Z %b %a")) elif args.time: print(datetime.now(timezone.utc).astimezone().strftime("=== %Y-%m-%d %H:%M:%S %z %b %a")) if args.cookie != '': print("Reading cookie") try: with open(args.cookie, 'r') as cookiefile: lastline = None for line in cookiefile: lastline = line if lastline is not None and lastline != '': cookieoffset = int(lastline, 0) if args.minimum_offset != '0': raise LogicError( "Both cookie %06X present and minimum offset %06X given" \ % (cookieoffset, int(args.minimum_offset, 0))) args.minimum_offset = ("0x%06X" % cookieoffset) else: print("Cookie file is empty!") except FileNotFoundError: print("Cookie file not found!") pass originalmaximumoffset = args.maximum_offset if args.maximum_offset[0] == "+": args.maximum_offset = "0x%06X" % (int(args.minimum_offset, 0) \ + int(args.maximum_offset[1:], 0) - 1) sys.stdout.reconfigure(line_buffering = True); if not args.no_version: scriptversion = subprocess.check_output( ['hg', 'id', '-i'], cwd = os.path.dirname(os.path.realpath(__file__))).decode('ascii').strip() print("ident86 version: hg %s" % scriptversion) try: if not objectinitialised: objectinitialised = True object = ldebug_class() object.setUp() object.do_read() if object.debug & 1: print(object.buffer) object.buffer = "" if not args.no_version: os.write(object.m, b"?version\r") object.do_read() if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger ?version command" search = re.search(r"(?m)^(lDebug .*?)[\r\n]*$", object.buffer) version = search.group(1) print("lDebug version: \"%s\"" % version) object.buffer = "" if args.version: raise VersionCommandNoException() os.write(object.m, b"uninstall paging\rr dco2 clr= 333\r") object.do_read() if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger configuration" object.buffer = "" if args.side_by_side: os.write(object.m, b"r DAO or= 200\r") object.do_read() if object.debug & 1: print(object.buffer) assert "^ Error" not in object.buffer, "Error in debugger disassembler configuration" object.buffer = "" amount = len(args.filenames) if amount < 2 or amount > 4: print("Usage: %s file1.bin file2.bin [file.tls [file.map]]" % sys.argv[0], file=sys.stderr) exit() print("Number of files: " + str(amount)) print("File 1: %s" % args.filenames[0]) print("File 2: %s" % args.filenames[1]) tlsfilename = None if amount >= 3: tlsfilename = args.filenames[2] print("Trace listing file: %s" % tlsfilename) mapfilename = None if amount >= 4: mapfilename = args.filenames[3] print("WarpLink map file: %s" % mapfilename) mapranges = () if mapfilename is None: mapranges = ([0x0, 0xFFFFffff],) else: with open(mapfilename, 'r') as mapfile: mapheaderpattern = re.compile(r'Detailed Segment Map') mapcolumnspattern = re.compile( r'^(\s*Name\s+)(Ovl#\s+)(Address\s+)(Length\s+)') patternaddress = re.compile( r'^([0-9A-Fa-f]{1,4})[Hh]?:([0-9A-Fa-f]{1,8})[Hh]?$') # $1 = segment, $2 = offset patternlength = re.compile( r'^([0-9A-Fa-f]{1,8})[Hh]?$') patternorigin = re.compile( r'^([0-9A-Fa-f]{1,16})[Hh]?$') maporiginpattern = re.compile(r'-- Program origin --') mapsectionssummarypattern = re.compile(r'-- Sections \(summary\) --') mapsectionscolumnspattern = re.compile( r'Vstart\s+Start\s+Stop\s+Length\s+Class') patternsection = re.compile(r"""(?xi) ^\s*[0-9A-Fa-f]+ # vstart \s+([0-9A-Fa-f]+) # start = $1 \s+[0-9A-Fa-f]+ # stop \s+([0-9A-Fa-f]+) # length = $2 \s+(progbits|nobits) # class = $3 \s+ """) mapsectionsdetailedorsymbols = re.compile( r'-- (Symbols|Sections \(detailed\)) --') indetailed = 0 gotcolumns = 0 inorigin = 0 insummary = 0 org = 0 gotorg = 0 for line in mapfile: match = re.search(mapheaderpattern, line) if match is not None: indetailed = 1 inorigin = 0 insummary = 0 continue match = re.search(maporiginpattern, line) if match is not None: indetailed = 0 inorigin = 1 insummary = 0 continue match = re.search(mapsectionssummarypattern, line) if match is not None: indetailed = 0 inorigin = 0 insummary = 1 continue if inorigin: if re.search(r'^\s*$', line): continue match = re.search(patternorigin, line) if match is None: raise InputError("Expected program origin in map file") if gotorg: raise InputError("Did not expect multiple origins in map file") org = hex64bitstosigned(match.group(1)) gotorg = 1 continue if insummary: if re.search(r'^\s*$', line): continue if not gotcolumns: match = re.search(mapsectionscolumnspattern, line) if match is None: raise InputError("Expected column header in map file") gotcolumns = 1 continue match = re.search(mapsectionsdetailedorsymbols, line) if match is not None: break match = re.search(patternsection, line) if match is None: raise InputError("Expected section line match in map file") address = (hex64bitstosigned(match.group(1)) - org) length = int(match.group(2), 16) bitsclass = match.group(3) if length and bitsclass == "progbits": mapranges += ([address, address + length - 1],) continue if indetailed: if not gotcolumns: match = re.search(mapcolumnspattern, line) if match is None: raise InputError("Expected column header in map file") columnsname = len(match.group(1)) columnsovl = len(match.group(2)) columnsaddress = len(match.group(3)) columnslength = len(match.group(4)) pattern = re.compile(r"""(?mx) ^(.{%u}) # $1 = name (.{%u}) # $2 = ovl (.{%u}) # $3 = address (.{%u}) # $4 = length """ \ % (columnsname, columnsovl, columnsaddress, columnslength)) gotcolumns = 1 continue if re.search(r'^\s*$', line): break match = re.search(pattern, line) if match is None: raise InputError("Expected segment line match in map file") textaddress = match.group(3).strip() textlength = match.group(4).strip() if re.search(r'\s', textaddress): raise InputError("Unexpected embedded blank in address column") if re.search(r'\s', textlength): raise InputError("Unexpected embedded blank in length column") matchaddress = re.search(patternaddress, textaddress) matchlength = re.search(patternlength, textlength) if matchaddress is None: raise InputError("Address column doesn't match") if matchlength is None: raise InputError("Length column doesn't match") segment = int(matchaddress.group(1), 16) offset = int(matchaddress.group(2), 16) length = int(matchlength.group(1), 16) if length: address = segment * 16 + offset mapranges += ([address, address + length - 1],) continue if (not len(mapranges)): print("Error: Map file not recognised!") mapranges = sorted(mapranges, key=lambda tuple: tuple[0]) print("Not merged map ranges:") for tuple in mapranges: print("%06X up to below %06X" % (tuple[0], tuple[1] + 1)) mergedranges = () idx = 0 while idx < len(mapranges): start = mapranges[idx][0] while idx + 1 < len(mapranges) \ and mapranges[idx][1] + 1 == mapranges[idx + 1][0]: idx += 1 end = mapranges[idx][1] mergedranges += ([start, end],) idx += 1 mapranges = mergedranges print("Merged map ranges:") for tuple in mapranges: print("%06X up to below %06X" % (tuple[0], tuple[1] + 1)) none_context = contextlib.nullcontext() with \ open(args.filenames[0], 'rb') as f1, \ open(args.filenames[1], 'rb') as f2, \ (open(tlsfilename, 'r', encoding='latin-1') if tlsfilename is not None else none_context) as tlsfile: if int(args.minimum_offset, 0) > int(args.maximum_offset, 0): print("Error: Minimum offset above maximum offset!") if args.pruef: sum1 = pruef(f1, "File 1") sum2 = pruef(f2, "File 2") elif args.repeat: sum1 = pruef(f1) sum2 = pruef(f2) if args.repeat: global priorsum1, priorsum2 if not repeating: priorsum1 = sum1 priorsum2 = sum2 else: if args.edit_file == 1: editsum = sum1 prioreditsum = priorsum1 originalsum = sum2 priororiginalsum = priorsum2 else: editsum = sum2 prioreditsum = priorsum2 originalsum = sum1 priororiginalsum = priorsum1 if not args.wlcalc_replace and editsum == prioreditsum: raise InputError("Checksum of file to edit didn't change!") if args.wlcalc_replace and editsum != prioreditsum: raise InputError("Checksum of file to edit did change!") if originalsum != priororiginalsum: raise InputError("Checksum of original file did change!") edited = False amountnodifferenceranges = 0 amountdifferencelines = 0 amountdifferencebytes = 0 earliestdifferenceflag = None maximum = None eof = 0 length = 0 prioroffset = 0 offset = 0 priorarray = [] array = [] ones = 0 zeroes = 0 foundfile = [None,] foundbase = None foundoffset = None foundlinenumber = None lastdisassembly = int(args.minimum_offset, 0) relocs1 = {} relocs2 = {} relocs1only = {} relocs2only = {} headersize = 0 header = f1.read(32) f1.seek(0, 0) if len(header) == 32 and \ (header[0:2] == b"MZ" or header[0:2] == b"ZM"): headersize = (header[8] + header[9] * 256) * 16 print("MZ executable header detected, size = %u bytes" % headersize) f2headersize = 0 f2header = f2.read(32) f2.seek(0, 0) if len(f2header) == 32 and \ (f2header[0:2] == b"MZ" or f2header[0:2] == b"ZM"): f2headersize = (f2header[8] + f2header[9] * 256) * 16 if not headersize == f2headersize: raise InputError("Header sizes mismatch") reloc1count = header[6] + header[7] * 256 reloc1offset = header[0x18] + header[0x19] * 256 reloc2count = f2header[6] + f2header[7] * 256 reloc2offset = f2header[0x18] + f2header[0x19] * 256 if not reloc1count == reloc2count: print("Error: Relocation counts mismatch! f1=%u f2=%u" \ % (reloc1count, reloc2count)) relocs1 = readreloctable(f1, reloc1offset, reloc1count, 1) relocs2 = readreloctable(f2, reloc2offset, reloc2count, 2) if (relocs1 != relocs2): print("Error: Relocation table mismatch!") relocs1only = {k: v for k, v in relocs1.items() if k not in relocs2.keys()} relocs2only = {k: v for k, v in relocs2.items() if k not in relocs1.keys()} for key in relocs1only: print("Reloc only in file 1 = %06Xh" % key) for key in relocs2only: print("Reloc only in file 2 = %06Xh" % key) alloc1 = getalloc(header) alloc2 = getalloc(f2header) if (alloc1 != alloc2): print(("Error: Allocation size mismatch!" + " f1=%04Xh f2=%04Xh delta=%04Xh") % (alloc1, alloc2, alloc1 - alloc2)) for ff in [ \ [0x0E, 'Init SS'], \ [0x10, 'Init SP'], \ [0x14, 'Init IP'], \ [0x16, 'Init CS'] ]: field = ff[0] name = ff[1] content1 = header[field] + 256 * header[field + 1] content2 = f2header[field] + 256 * f2header[field + 1] if content1 != content2: print(("Error: Header field %s at %02Xh mismatch!" + " f1=%04Xh f2=%04Xh delta=%04Xh") % (name, field, content1, content2, content1 - content2)) originaloffset = args.offset if args.offset[0] == "+": args.offset = "0x%06X" % (headersize \ + int(args.offset[1:], 0)) args.offset = re.sub(r'^0x-', '-0x', args.offset) if args.skip_header: lastdisassembly = max(lastdisassembly, headersize) for tuple in mapranges: tuple[0] += int(args.offset, 0) tuple[1] += int(args.offset, 0) invertedmapranges = () idx = 0 priorendrange = 0 while idx <= len(mapranges): if idx == len(mapranges): startrange = 0xFFFFffff + 1 endrange = None else: tuple = mapranges[idx] startrange = tuple[0] endrange = tuple[1] if object.debug & (512 * 1024 + 1): er = 0 if endrange is not None: er = endrange print("startrange=%06Xh=%u endrange=%06Xh=%u" % (startrange, startrange, er, er)) if startrange != 0 and startrange - 1 != priorendrange: if object.debug & (512 * 1024 + 1): print("priorendrange=%06Xh=%u startrange=%06Xh=%u" % (priorendrange, priorendrange, startrange - 1, startrange - 1)) invertedmapranges += ([priorendrange, startrange - 1],) priorendrange = endrange if priorendrange is not None: priorendrange += 1 idx += 1 if object.debug & (512 * 1024 + 1): print("###", end="") print(invertedmapranges); autoreached = False applyreloc = int(args.apply_reloc, 0) carry1 = 0 carry2 = 0 if args.pattern is not None: for combinedpattern in args.pattern: splitpatterns = combinedpattern.split("::") if not (len(splitpatterns) >= 1 and len(splitpatterns) <= 3): raise InputError("Invalid -p pattern in \"%s\"" % combinedpattern) forced = [] if args.forced is not None: for forceditem in args.forced: splititem = forceditem.split("::") if not (len(splititem) >= 1 and len(splititem) <= 2): raise InputError("Invalid -F pattern in \"%s\"" % forceditem) first = int(splititem[0], 0) if len(splititem) == 1: forced.append(first) else: second = int(splititem[1], 0) for idx in range(first, second + 1): forced.append(idx) while not eof: buffer1 = bytearray(f1.read(buffersize)) buffer2 = bytearray(f2.read(buffersize)) length = min(len(buffer1), len(buffer2)) carry1 = applyrelocations(buffer1, offset - headersize, length, relocs1, carry1) carry2 = applyrelocations(buffer2, offset - headersize, length, relocs2, carry2) priorarray.extend(array) array = [] array.extend([0] * length) for idx in range(length): value1 = buffer1[idx] value2 = buffer2[idx] if value1 != value2 or idx + offset in forced: array[idx] = 1 zeroes = 0 headernote = "" if offset + idx < headersize: headernote = " (header)" if args.skip_header: continue if offset + idx < int(args.minimum_offset, 0): continue if offset + idx > int(args.maximum_offset, 0): continue ones += 1 amountdifferencebytes += 1 print("%06X first:%02X != second:%02X%s" % (offset + idx, value1, value2, headernote)) auto = int(args.auto_length, 0) if auto and ones >= auto: autoreached = True print("Automatic difference length exceeded in bytes display!") args.maximum_offset = \ ("0x%06X" % (offset + idx)) break else: zeroes += 1 if zeroes >= 16 and ones: ones = 0 handlerange(offset) if len(buffer1) < buffersize: if len(buffer2) >= len(buffer1): print("EOF1 reached at %u bytes" % f1.tell()) eof |= 1 if len(buffer2) < buffersize: if len(buffer1) >= len(buffer2): print("EOF2 reached at %u bytes" % f2.tell()) eof |= 2 prioroffset = offset offset += length if ones: ones = 0 idx = length handlerange(prioroffset) f1.lastread = f1.tell() f2.lastread = f2.tell() f1.seek(0, 2) f2.seek(0, 2) f1length = f1.tell() f2length = f2.tell() if eof == 2 or eof == 1: longer = f1length delta = f1length - f2length if eof == 1: delta = - delta longer = f2length plural = "" if delta != 1: plural = "s" print("File %u (%u bytes) is %u byte%s longer than file %u" % (eof ^ 3, longer, delta, plural, eof)) longerbuffer = (buffer1, buffer2)[(eof ^ 3) - 1] longerbuffer = longerbuffer[length:buffersize] longerfile = (f1, f2)[(eof ^ 3) - 1] longerfile.seek(longerfile.lastread, 0) trailersize = int(args.trailer_size, 0) trailerlength = 0 if len(longerbuffer) < trailersize and len(longerbuffer) < delta: trailerlength = min(trailersize - len(longerbuffer), \ delta - len(longerbuffer)) if object.debug & 65536: print("TSz=%d TLen=%d BLen=%d Len=%d" \ % (trailersize, trailerlength, len(longerbuffer), length)) if trailerlength: trailer = longerfile.read(trailerlength) if object.debug & 65536: print("TLen=%d Len=%d" \ % (trailerlength, len(trailer))) if not len(trailer) == trailerlength: raise InputError("Unable to read trailer") longerbuffer += trailer else: longerbuffer = longerbuffer[0:trailersize] trailerdisplayed = 0 for byte in longerbuffer: if offset < int(args.minimum_offset, 0): continue if offset > int(args.maximum_offset, 0): continue amountdifferencebytes += 1 amountdifferencelines += 1 trailerdisplayed += 1 if eof == 2: print("%06X first:%02X != second:eof" % (offset, byte)) else: print("%06X first:eof != second:%02X" % (offset, byte)) offset += 1 if trailerdisplayed < delta: print("%06X %u bytes omitted" % \ (offset, delta - trailerdisplayed)) amountomitted = max(0, min(delta - trailerdisplayed, \ int(args.maximum_offset, 0) - offset)) if object.debug & 65536: print("%d" % amountomitted) amountdifferencebytes += amountomitted amountdifferencelines += amountomitted else: print("Files are the same length (%u bytes)" % f1length) print("Amount different bytes: %u" % amountdifferencebytes) print("Amount different lines: %u" % amountdifferencelines) print("Amount not different ranges: %u" % amountnodifferenceranges) if earliestdifferenceflag is not None: print("Earliest difference at %06X" % earliestdifferenceflag) if repeating: if priorearliestdifferenceflag >= earliestdifferenceflag: raise InputError("No progress made by last edit!" + (" (prior=%06X)" % (priorearliestdifferenceflag))) priorearliestdifferenceflag = earliestdifferenceflag repeating = False if args.repeat and edited: globalrepeat = True repeating = True if args.cookie != '': args.minimum_offset = '0' args.maximum_offset = originalmaximumoffset args.offset = originaloffset except VersionCommandNoException: pass finally: if not repeating and objectinitialised: objectinitialised = False object.tearDown() if __name__ == '__main__': priorearliestdifferenceflag = None objectinitialised = False object = None globalrepeat = True repeating = False try: while globalrepeat: globalrepeat = False ident86main() finally: if objectinitialised: objectinitialised = False object.tearDown()