from __future__ import annotations import collections.abc import datetime import typing from warnings import deprecated # type: ignore import jpype # type: ignore import jpype.protocol # type: ignore import ghidra.pcode.emu.jit import ghidra.pcode.exec_ import ghidra.program.model.address import ghidra.program.model.lang import ghidra.program.model.listing import ghidra.program.model.pcode import java.lang # type: ignore import java.util # type: ignore class JitPassageDecoder(java.lang.Object): """ The decoder of a :obj:`JitPassage` to support JIT-accelerated p-code emulation. When the emulator encounters an address (and contextreg value) that it has not previously translated, it must decode a passage seeded at that required entry point. It must then translate the passage and collects all the resulting entry points, and finally invoke the passage's :meth:`run ` method for the required entry point. ****************** Decoding a Passage ****************** Decode starts with a single seed, which is the entry point required by the emulator. As such, that seed *must* be among the entry points exported by the translator. Decode occurs one stride at a time. Starting with the seed, we decode a stride by disassembling linearly until: 1) We encounter an instruction without fall through, 2) there's already an entry point to a translated passage at an encountered address, or 3) a user injection fails to specify control flow. Case 1 is the normal expected case. For example, when the decoder encounters an unconditional branch, the stride is terminated. Case 2 is meant to reduce duplicative translations, but it does come at some cost during decode time. Suppose execution branches into the middle of a previously translated basic block. (Note that basic blocks are only broken apart using branches *in the same passage*, so it is possible some branch encountered later would jump into another passage's basic block.) That previously translated passage will not have exposed an entry point at that branch target, so the emulator will begin decoding using the branch target as the seed. Ideally, the resulting passage will consist of a single stride that terminates at an existing entry point. The emulator will translate and execute the passage, which should exit at that entry point, where the emulator can then continue execution. Case 3 is just to ensure execution does not get caught in a translated infinite loop. There will still be an infinite loop, but it can be interrupted while execution is in the emulator's logic rather than the translated logic. As the stride decoder processes each instruction, it interprets its p-code, along with any generated by user injects, to collect branch targets. For direct branches (:obj:`branch `, :obj:`cbranch `, and :obj:`call `), the target address (and appropriate contextreg value) is added to the queue of seeds, unless that target is already decoded in this passage. A bit of control flow analysis is required to determine whether each instruction (with user injects) has fall through. We borrow the :obj:`BlockSplitter` from the :obj:`JitControlFlowModel` to accomplish this. We append a "probe" p-code op at the very end, and then once we have the (miniature) control flow graph, we check if there's a path from instruction start to the probe op. If there is, then we can fall through, so decode proceeds to the next instruction. If not, the stride is terminated, so the decoder starts a new stride at the next seed, unless we've met the p-code op, instruction, or stride :obj:`quota `. The seed queue is a list of :obj:`ExtBranch` records. Each stride is decoded by removing a seed from that queue, decoding instructions, emitting ops, and then creating an :obj:`IntBranch` record targeting the first op of the newly-decoded instruction. The :meth:`from ` field is taken from the seed :obj:`ExtBranch` record. Decode will likely terminate before this queue is emptied, in which case, those remaining external branches will become part of the passage's :meth:`branches `. Direct branches to instructions already included in the passage, p-code relative branches, and queued external branches to instructions which have since been decoded all become :obj:`IntBranch` records, too. For indirect branches (:obj:`branchind `, :obj:`callind `, and :obj:`return `), we create :obj:`IndBranch` records. For error cases (e.g., :obj:`unimplemented `), we create :obj:`ErrBranch` records. .. admonition:: Implementation Note The process described above is actually implemented in :obj:`DecoderForOnePassage`. This class just keeps the configuration and some other trappings, and instantiates an actual decoder upon requesting a seed. """ class_: typing.ClassVar[java.lang.Class] def __init__(self, thread: ghidra.pcode.emu.jit.JitPcodeThread): """ Construct a passage decoder :param ghidra.pcode.emu.jit.JitPcodeThread thread: the thread whose instruction decoder, context, and userop library to use. """ @typing.overload def decodePassage(self, seed: ghidra.program.model.address.Address, ctxIn: ghidra.program.model.lang.RegisterValue, maxOps: typing.Union[jpype.JInt, int]) -> ghidra.pcode.emu.jit.JitPassage: """ Decode a passage starting at the given seed :param ghidra.program.model.address.Address seed: the seed address :param ghidra.program.model.lang.RegisterValue ctxIn: the seed contextreg value :param jpype.JInt or int maxOps: the maximum-ish number of p-code ops to emit :return: the decoded passage :rtype: ghidra.pcode.emu.jit.JitPassage .. seealso:: | :obj:`.decodePassage(AddrCtx, int)` """ @typing.overload def decodePassage(self, seed: ghidra.pcode.emu.jit.JitPassage.AddrCtx, maxOps: typing.Union[jpype.JInt, int]) -> ghidra.pcode.emu.jit.JitPassage: """ Decode a passage starting at the given seed We provide a ``maxOps`` parameter so that the configured :meth:`option ` can be overridden. In particular, the bytecode emitter may exceed the maximum size of a Java method, in which case we must abort, re-decode with fewer ops, and retry. Whether this back off should persist in the configuration is yet to be determined. Output size can vary wildly depending on the number of basic blocks, scope transitions, nature of the ops, etc. We ought to be able to provide a reasonable default value that mostly avoids retries, because each retry essentially wastes an entire JIT translation. On the other hand, if we choose too small a value, we lose some of the benefits of translating the control flow and keeping variables in JVM locals. :param ghidra.pcode.emu.jit.JitPassage.AddrCtx seed: the required entry point, where decode will start :param jpype.JInt or int maxOps: the maximum-ish number of p-codes to emit :return: the decoded passage :rtype: ghidra.pcode.emu.jit.JitPassage """ @typing.type_check_only class DecoderExecutor(ghidra.pcode.exec_.PcodeExecutor[java.lang.Object], ghidra.program.model.lang.DisassemblerContextAdapter): """ The p-code interpreter used during passage decode Aside from branches, this interpreter simply logs each op, so that they get collected into the greater stride and passage. It does "rewrite" the ops, so that we can easily recover the input context, especially when the op is emitted from a user inject. For branches, this interpreter creates the appropriate :obj:`Branch` records and notifies the passage decoder of new seeds. This executor also implements the :obj:`DisassemblerContext` to track context changes, namely uses of ``globalset``. This is kept in :obj:`.futCtx`. **TODO**: Should :obj:`.futCtx` be moved into the passage decoder to ensure it persists for more than a single instruction? I'm not sure whether or not that is already taken care of by the :obj:`Disassembler`. .. admonition:: Implementation Note I had considered using a :obj:`JitDataFlowState` here, but that's Not a Good Idea, because a stride is not generally a *basic block*. A "stride" is just a contiguous run of instructions with fall-through. If there is a jump into the middle of it, any value analysis (e.g., constant folding) would be meaningless. Were we to put this in there, the temptation may be to have userop libraries attempt constant resolution, esp., for syscall numbers. While that may work, if only because syscall numbers are conventionally set in the same basic block as the invocation, there's no guarantee that's the case. And there may be other use cases where this is totally wrong. Instead, we should use as barren an executor here as possible. We do incorporate injects here, because they may affect control flow, which the decoder must consider. .. admonition:: Implementation Note **WARNING**: This executor has no :obj:`state ` object. Care must be taken to ensure we override any method that assumes we have one, and that we don't invoke any method from the superclass that assumes we have one. """ class_: typing.ClassVar[java.lang.Class] def checkFallthroughAndAccumulate(self, from_: ghidra.pcode.exec_.PcodeProgram) -> ghidra.pcode.emu.jit.JitPassage.Reachability: """ After p-code interpretation, check if the instruction has fall through, notify the stride decoder of the instruction's ops, and notify the passage of the instruction's branches. To determine whether there's fall through, this performs a miniature control flow analysis on just this step's p-code ops. This is required because a user inject can be very complex, and need not obey all of the usual control flow checks imposed by the Sleigh semantic compiler. In particular :meth:`Instruction.hasFallthrough() ` is not sufficient, for at least two reasons: 1) The aforementioned user inject possibilities, 2) We do not consider a :obj:`call ` or :obj:`callind ` as having fall through. To use control flow analysis as a means of checking for fall through, we append a special "probe" :obj:`ExitPcodeOp` along with an :obj:`ExtBranch` record to :obj:`nowhere `. The probe thus serves the secondary purpose of preventing any complaints from the analyzer about unterminated control flow. We then perform the analysis, borrowing :obj:`BlockSplitter` from :obj:`JitControlFlowModel`. In practice, this seems fast enough. Because the splitter keeps the blocks in the original order, the first op will certainly be in the first block, and the probe op will certainly be in the last block. We perform a simple reachability test between the two. The step has fall through if and only if a path is found. :param ghidra.pcode.exec_.PcodeProgram from: the instruction's or inject's p-code :return: the reachability of the fall-through flow :rtype: ghidra.pcode.emu.jit.JitPassage.Reachability """ def execute(self, program: ghidra.pcode.exec_.PcodeProgram): """ Interpret the given program with the passage decoder's userop library :param ghidra.pcode.exec_.PcodeProgram program: the p-code to interpret """ def takeTargetContext(self, target: ghidra.program.model.address.Address) -> ghidra.pcode.emu.jit.JitPassage.AddrCtx: """ Derive the contextreg value at the given target address (branch or fall through). An instruction's constructors may use ``globalset`` to place context changes at specific addresses. Those changes are collected by :meth:`setFutureRegisterValue(Address, RegisterValue) <.setFutureRegisterValue>` through some chain of method invocations started by :meth:`setInstruction(PseudoInstruction) <.setInstruction>`. When the interpreter encounters a branch op, that op will specify the target address. We must also derive the context for that branch. This is the pre-computed "flow" context, but now accounting for ``globalset`` at the target address. :param ghidra.program.model.address.Address target: the target address :return: the target address and contextreg value :rtype: ghidra.pcode.emu.jit.JitPassage.AddrCtx """ @typing.type_check_only class DecoderForOnePassage(java.lang.Object): """ The decoder for a single passage This is a sort of "mutable" passage or passage "builder" that is used while the passage is being decoded. Once complete, this provides an immutable (or at least it's supposed to be) decoded :obj:`Passage`. """ class_: typing.ClassVar[java.lang.Class] class DecoderForOneStride(java.lang.Object): """ The decoder for a single stride. This starts at a given seed and proceeds linearly until it hits an instruction without fall through. It may also stop if it encounters an existing entry point or an erroneous user inject. .. seealso:: | :obj:`JitPassageDecoder` """ @typing.type_check_only class StepResult(java.lang.Record): """ The result of decoding an instruction This may also represent an error encountered while trying to decode an instruction. """ class_: typing.ClassVar[java.lang.Class] def equals(self, o: java.lang.Object) -> bool: ... def executor(self) -> DecoderExecutor: ... def hashCode(self) -> int: ... def program(self) -> ghidra.pcode.exec_.PcodeProgram: ... def toString(self) -> str: ... class_: typing.ClassVar[java.lang.Class] def __init__(self, decoder: JitPassageDecoder, passage: DecoderForOnePassage, start: ghidra.pcode.emu.jit.JitPassage.AddrCtx): """ Construct a stride decoder :param JitPassageDecoder decoder: the thread's passage decoder :param DecoderForOnePassage passage: the decoder for this specific passage :param ghidra.pcode.emu.jit.JitPassage.AddrCtx start: the seed to start this stride """ def decode(self) -> DecodedStride: """ Decode the stride. :return: the decoded stride :rtype: DecodedStride """ class DecoderUseropLibrary(ghidra.pcode.exec_.AnnotatedPcodeUseropLibrary[java.lang.Object]): """ The decoder's wrapper around the emulator's userop library This library serves two purposes: 1) to override :meth:`PcodeEmulationLibrary.emu_exec_decoded() ` and :meth:`PcodeEmulationLibrary.emu_skip_decoded() `, and 2) to check and inline p-code userops that :meth:`allow ` it. We accomplish the first purpose simply by adding the two userops using the usual annotations. The two built-in userops regarding the decoded instruction are easily inlinable, so we will mark them as such. Note, however, that they are separate from the wrappers we mention for the second purpose (inlining), and so we must implement that inlining in the actual userop. We still mark them for informational purposes and because the translator needs to know. We accomplish the second purpose of inlining by accepting the emulator's userop library and individually wrapping each of its userops, excluding the two we override. We allow each userop's attributes to pass through, but when executed, we check if the userop allows inlining. If so, then we feed the userop's p-code into the decoder's interpreter. This effectively inlines the op, control flow ops and all, into the passage. Note we do not actually replace the :obj:`callother ` op, for bookkeeping purposes. Instead we will map it to a :obj:`nop ` during translation. """ @typing.type_check_only class WrappedUseropDefinition(ghidra.pcode.exec_.PcodeUseropLibrary.PcodeUseropDefinition[java.lang.Object]): """ The wrapper around one of the emulator's userops """ class_: typing.ClassVar[java.lang.Class] def __init__(self, rtOp: ghidra.pcode.exec_.PcodeUseropLibrary.PcodeUseropDefinition[jpype.JArray[jpype.JByte]]): """ Wrap the given userop :param ghidra.pcode.exec_.PcodeUseropLibrary.PcodeUseropDefinition[jpype.JArray[jpype.JByte]] rtOp: the actual userop, as defined by the user or emulator """ class_: typing.ClassVar[java.lang.Class] def __init__(self, rtLib: ghidra.pcode.exec_.PcodeUseropLibrary[jpype.JArray[jpype.JByte]]): """ Wrap the given userop library :param ghidra.pcode.exec_.PcodeUseropLibrary[jpype.JArray[jpype.JByte]] rtLib: the actual library provided by the user or emulator """ def emu_exec_decoded(self, executor: ghidra.pcode.exec_.PcodeExecutor[java.lang.Object]): """ The replacement for :meth:`PcodeEmulationLibrary.emu_exec_decoded() `. The one built into the emulator would have the thread interpret the decoded instruction directly. While this might "work," it totally missed the purpose of JIT translation. We instead inline the userop's p-code into the rest of the passage. We accomplish this by having the decoder interpret the p-code instead. We also need to ensure the decoded instruction is added into the passage. Note that the :obj:`callother ` op will be mapped to a :obj:`nop ` during translation because we have set ``canInline``. :param ghidra.pcode.exec_.PcodeExecutor[java.lang.Object] executor: the decoder's executor """ def emu_skip_decoded(self, executor: ghidra.pcode.exec_.PcodeExecutor[java.lang.Object]): """ The replacement for :meth:`PcodeEmulationLibrary.emu_skip_decoded() `. The one built into the emulator would have the thread drop and skip the decoded instruction directly. This would not have the intended effect, because the decoder is the thing that needs to skip and advance to the next address. We instead "inline" nothing, but we must still decode the instruction. Because the executor provides the decode routine, it can internally work out fall through. We will *not* add the instruction to the passage, though, because we will not have the executor interpret any of the instructon's p-code. As for fall through, the :meth:`DecoderExecutor.checkFallthroughAndAccumulate(PcodeProgram) ` routine just does its usual. If the inject falls through, :meth:`DecoderExecutor.getAdvancedAddress() ` considers the decoded instruction, even though it was never interpreted. Note that the :obj:`callother ` op will still be mapped to a :obj:`nop ` during translation because we have set ``canInline``. :param ghidra.pcode.exec_.PcodeExecutor[java.lang.Object] executor: the decoder's executor """ @typing.type_check_only class DecodedStride(java.lang.Record): """ A list of contiguous instructions connected by fall through, along with their emitted p-code ops .. seealso:: | :obj:`JitPassageDecoder` """ class_: typing.ClassVar[java.lang.Class] def equals(self, o: java.lang.Object) -> bool: ... def hashCode(self) -> int: ... def instructions(self) -> java.util.List[ghidra.program.model.listing.Instruction]: ... def ops(self) -> java.util.List[ghidra.program.model.pcode.PcodeOp]: ... def start(self) -> ghidra.pcode.emu.jit.JitPassage.AddrCtx: ... def toString(self) -> str: ... __all__ = ["JitPassageDecoder", "DecoderExecutor", "DecoderForOnePassage", "DecoderForOneStride", "DecoderUseropLibrary", "DecodedStride"]