mirror/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2026-08-08 07:40:39 -09:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/GP-6921_d-millar_struct_recovery_RB260706'

Browse Source
This commit is contained in:
Ryan Kurtz
2026-07-17 05:01:02 -04:00
parent b24b7fd897 e1dddb3ec0
commit d23de7587d
10 changed files with 1786 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Ghidra/Features/BSim/certification.manifest
View File

@@ -41,6 +41,7 @@ src/main/help/help/topics/BSimSearchPlugin/images/BSimOverviewResults.png||GHIDR
src/main/help/help/topics/BSimSearchPlugin/images/BSimResultsProvider.png||GHIDRA||||END|
src/main/help/help/topics/BSimSearchPlugin/images/BSimSearchDialog.png||GHIDRA||||END|
src/main/help/help/topics/BSimSearchPlugin/images/ManageServersDialog.png||GHIDRA||||END|
src/main/help/help/topics/StructureRecovery/StructureRecovery.html||GHIDRA||||END|
src/main/resources/bsim.log4j.xml||GHIDRA||||END|
src/main/resources/images/checkmark_yellow.gif||GHIDRA||||END|
src/main/resources/images/connected.png||GHIDRA||||END|

22
Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml
View File

@@ -170,6 +170,28 @@
text="BSim Command (bsim)"
target="help/topics/BSim/CommandLineReference.html#BSimCommand"/>
</tocdef>
<tocdef id="StructureRecovery" sortgroup="f"
text="Structure Recovery Using BSim"
target="help/topics/StructureRecovery/StructureRecovery.html" >
<tocdef id="TagStructureUse"
sortgroup="a"
text="Tag structure use"
target="help/topics/StructureRecovery/StructureRecovery.html#TagStructureUse"/>
<tocdef id="MatchFunctions"
sortgroup="b"
text="Match functions"
target="help/topics/StructureRecovery/StructureRecovery.html#MatchFunctions"/>
<tocdef id="GenerateStructure"
sortgroup="c"
text="Generate structure"
target="help/topics/StructureRecovery/StructureRecovery.html#GenerateStructure"/>
<tocdef id="Options"
sortgroup="z"
text="Options"
target="help/topics/StructureRecovery/StructureRecovery.html#Options"/>
</tocdef>
</tocdef>
</tocref>
</tocroot>

203
Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html Normal file
View File

@@ -0,0 +1,203 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Structure Recovery Using BSim</title>
<link rel="stylesheet" type="text/css" href="help/shared/DefaultStyle.css">
<link rel="stylesheet" type="text/css" href="../../shared/languages.css">
<meta name="generator" content="DocBook XSL Stylesheets V1.79.1">
</head>
<body><div class="chapter">
<div class="titlepage"><div><div><h1 class="title">
<a name="StructureRecovery"></a>Structure Recovery Using BSim</h1></div></div></div>
<p>
The StructureRecovery plugin allows users to build a structure in a target program based on the
field uses in a source program. The process involves three steps. First, a source program is
mined for field references from a known version of the structure to recover the use addresses
and offsets. Second, functions in the source program bookmarked by field use are matched to
functions in the target program using the local variation of BSim, which does not require a database.
Third, BSim is used to create 'pinnings' between the data-flow graphs for each pair of functions
which identify vertex-vertex correspondences using the decoompiler's diffing algorithm.
Vertices with addresses from the first step are examined to identify offset information, which
then becomes input to the construction of a new structure in the target program.
</p>
<p>
The success of this process depends on a number of factors. There are cases, for instance, in
which nested structures may be missed by the structure use task, and, of course, certain fields
may not be referenced at all. BSim, here as elsewhere, depends on a number of parameters that
determine what constitutes a match, which are exposed as options. Unmatched functions will
potentially result in gaps in the structure. Finally, the pinning comparison attempts to reconcile
disparate pcode representations, but may fail in more complicated scenarios. When successful,
the resulting structure will be stored in the target program's DataTypeManager as "RECOVERED_structure_name".
</p>
<p>
Each of the three tasks is relatively expensive. It may be necessary to save results from each
across sessions. This is done using bookmarks (viewable from the listing or the Bookmarks viewer), which
capture the structure use and function-to-function matches. The former are represented as
"USES_structure_name_offset:address"; the latter as "source function id&rarr;target function id".
The function-match bookmarks will be present in both the source and target programs.
</p>
<p>
ALL tasks should run with the source program active.
</p>
<div class="section">
<div class="titlepage"><div><div><h2 class="title" style="clear: both">
<a name="MainActions"></a>Principal Actions (Tools &rarr; Structure Recovery)</h2></div></div></div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="TagStructureUse"></a>Tag Structure Use</h3></div></div></div>
<p>
Locates all non-zero field uses within a program for a specified structure.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MatchFunctions"></a>Match Functions</h3></div></div></div>
<p>
Uses BSim to match functions with structure use in the source program to similar functions
in a target program.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="GenerateStructure"></a>Generate structure</h3></div></div></div>
<p>
Compares data-flow 'pinnings' for matched functions and uses the resulting offsets to
reconstruct an approximate match to the original structure for the target progam.
</p>
</div>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div><h2 class="title" style="clear: both">
<a name="SecondaryActions"></a>Secondary Actions (Tools &rarr; Structure Recovery)</h2></div></div></div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="StructureUses"></a>Structure Uses: map to bookmarks</h3></div></div></div>
<p>
Converts (persistent) bookmarks containing structure-use information, typically generated
by the 'Tag Structure Use' action, to a (non-persistent) map and vice-versa.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="FunctionMatches"></a>Function Matches: map to bookmarks</h2></div></div></div>
<p>
Converts (persistent) bookmarks containing function-function matches, typically generated
by the 'Match Functions' action, to a (non-persistent) map and vice-versa.
</p>
<div class="section">
<div class="titlepage"><div><div><h2 class="title" style="clear: both">
<a name="Options"></a>Tool Options (Options/Structure Recovery)</h2></div></div></div>
<p>
Options governing various aspects of the structure recovery.
</p>
<div class="titlepage"><div><div><h3 class="title">
<a name="StructureToRecover"></a>Structure to recover</h3></div></div></div>
<p>
The full path to the data type to be replicated ('Tag Structure Use' will query if empty).
</p>
</div>
<div class="titlepage"><div><div><h3 class="title">
<a name="AddStructureUseBookmarks"></a>Add structure-use bookmarks</h3></div></div></div>
<p>
Should bookmarks by generated by 'Tag Structure Use'.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="AddFunctionMatchBookmarks"></a>Add function-match bookmarks</h3></div></div></div>
<p>
Should bookmarks by generated by 'Match Functions'.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MatchConfidenceLowerBound"></a>Match-confidence lower bound</h3></div></div></div>
<p>
Used by 'Match Functions' (see BSim documentation for details).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MatchSimilarityLowerBound"></a>Match-similarity lower bound</h3></div></div></div>
<p>
Used by 'Match Functions' (see BSim documentation for details).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MatchSimilarityUpperBound"></a>Match-similarity upper bound</h3></div></div></div>
<p>
Used by 'Match Functions' (see BSim documentation for details).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="SelfSignificanceBound"></a>Self-significance bound</h3></div></div></div>
<p>
Used by 'Match Functions' (see BSim documentation for details).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MinConfidence"></a>Min confidence</h3></div></div></div>
<p>
Used by 'Match Functions' (see BSim documentation for details).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MaxStructureOffset"></a>Max structure offset</h3></div></div></div>
<p>
The largest offset considered valid by 'Generate structure' (prevents confusion
between offsets and addresses).
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="ExcludedOffsets"></a>Excluded offsets (hex)</h3></div></div></div>
<p>
Comma-separated hex values to be ignored by 'Generate structure'. Principally used when
analyzing a randomized structure that amalgamates randomized fields in a common block,
e.g. 0x40 for the Linux task_struct randomized with symbols.
</p>
</div>
<div class="sect2">
<div class="titlepage"><div><div><h3 class="title">
<a name="MatchOn"></a>Match on...</h3></div></div></div>
<p>
Should the function-function matching rely on solo matches, solo matches plus matches
exceeding the 'Min confidence', or should a full maximum-weight bipartite match be
computed.
</p>
</div>
</div>
</dl></div>
</div>
</div></body>
</html>

400
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java Normal file
View File

@@ -0,0 +1,400 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.gui.structs;
import java.util.*;
import java.util.Map.Entry;
import db.Transaction;
import ghidra.app.decompiler.*;
import ghidra.app.services.ConsoleService;
import ghidra.features.codecompare.graphanalysis.DataVertex;
import ghidra.features.codecompare.graphanalysis.Pinning;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.*;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.Program;
import ghidra.program.model.pcode.*;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
public class ComparePinningsTask extends Task {
private StructureRecoveryPlugin plugin;
private ConsoleService console;
private static int DECOMPILER_TIMEOUT = 60;
private final Program currentProgram;
private final Program targetProgram;
private DecompInterface currentInterface;
private DecompInterface targetInterface;
private Map<Program, Map<Function, DecompileResults>> decompilerResults =
new HashMap<>();
private Map<Long, Map<Long, Float>> results = new HashMap<>();
private Map<String, Long> offsets;
private Map<String, AddressSet> addressesByField;
private Map<Function, Set<Function>> functionMap;
private long maxOffset = 0x100000;
private record PinningMatch(float weight, int sourceOffset, int targetOffset) {
@Override
public String toString() {
return Integer.toHexString(sourceOffset) + ":" + Integer.toHexString(targetOffset) +
" (" + weight + ")";
}
}
public ComparePinningsTask(StructureRecoveryPlugin plugin) {
super("Compare Pinnings", true, false, false);
this.plugin = plugin;
this.console = plugin.getConsole();
this.currentProgram = plugin.getCurrentProgram();
this.targetProgram = plugin.getTargetProgram();
this.offsets = plugin.getOffsets();
this.addressesByField = plugin.getAddressesByField();
this.functionMap = plugin.getFunctionMap();
this.maxOffset = plugin.getMaxOffset();
}
@Override
public void run(TaskMonitor monitor) {
String taskName = getTaskTitle();
try {
Thread.currentThread().setName(taskName);
console.addMessage(taskName, "Running...");
initDecompiler();
comparePinnings(monitor);
console.addMessage(taskName, "Finished!");
}
catch (Exception e) {
if (!monitor.isCancelled()) {
Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName +
"\n" + e.getClass().getName() + ": " + e.getMessage(), e);
console.addErrorMessage("", "Error running task: " + taskName);
console.addException(taskName, e);
}
}
finally {
cleanupDecompiler();
}
}
protected void comparePinnings(TaskMonitor monitor) throws CancelledException {
monitor.setMessage("Comparing pinnings...");
monitor.setIndeterminate(false);
monitor.initialize(functionMap.size());
for (Function left : functionMap.keySet()) {
monitor.checkCancelled();
Set<Function> tgtSet = functionMap.get(left);
for (Function right : tgtSet) {
monitor.checkCancelled();
DecompileResults currentResults =
getDecompilerResults(currentInterface, left, monitor);
HighFunction currentHFunc = currentResults.getHighFunction();
if (currentHFunc == null) {
continue;
}
DecompileResults targetResults =
getDecompilerResults(targetInterface, right, monitor);
HighFunction targetHFunc = targetResults.getHighFunction();
if (targetHFunc == null) {
continue;
}
Pinning pin =
Pinning.makePinning(currentHFunc, targetHFunc, false, false, true,
TaskMonitor.DUMMY);
for (String fieldName : offsets.keySet()) {
Long offset = offsets.get(fieldName);
AddressSet addresses = addressesByField.get(fieldName);
results.putIfAbsent(offset, new HashMap<>());
analyzePinning(pin, offset, addresses);
}
}
monitor.increment();
}
//printResults();
buildStructure();
}
private void initDecompiler() {
DecompileOptions options = new DecompileOptions();
currentInterface = new DecompInterface();
currentInterface.setOptions(options);
currentInterface.openProgram(currentProgram);
targetInterface = new DecompInterface();
targetInterface.setOptions(options);
targetInterface.openProgram(targetProgram);
}
private void cleanupDecompiler() {
if (currentInterface != null) {
currentInterface.dispose();
}
if (targetInterface != null) {
targetInterface.dispose();
}
decompilerResults.clear();
}
public DecompileResults getDecompilerResults(DecompInterface ifc, Function func,
TaskMonitor monitor) {
return decompilerResults
.computeIfAbsent(ifc.getProgram(), _ -> new HashMap<>())
.computeIfAbsent(func, f -> ifc.decompileFunction(f, DECOMPILER_TIMEOUT, monitor));
}
private void analyzePinning(Pinning pin, long offset, AddressSet addresses) {
Map<DataVertex, DataVertex> originalPinMap = pin.getPinMap();
Map<PcodeOp, DataVertex> op2vertex = new HashMap<>();
Map<PcodeOp, DataVertex> opToVertexMap = new HashMap<>();
for (DataVertex lv : originalPinMap.keySet()) {
PcodeOpAST op = lv.getOp();
if (op != null && !opToVertexMap.containsKey(op)) {
opToVertexMap.put(op, originalPinMap.get(lv));
op2vertex.put(op, lv);
}
}
for (PcodeOp lop : opToVertexMap.keySet()) {
DataVertex rv = opToVertexMap.get(lop);
DataVertex lv = op2vertex.get(lop);
if (addresses != null && addresses.contains(lop.getSeqnum().getTarget())) {
compareConstants(lv, rv, offset);
}
}
}
private void compareConstants(DataVertex vertL, DataVertex vertR, long offset) {
Set<Varnode> constantsL = new LinkedHashSet<>();
collectConstantsByOperand(vertL.getOp(), new HashSet<>(), constantsL,
vertL.getDepthPaired());
Set<Varnode> constantsR = new LinkedHashSet<>();
collectConstantsByOperand(vertR.getOp(), new HashSet<>(), constantsR,
vertR.getDepthPaired());
Set<Long> uniqL = extractConstants(constantsL);
Set<Long> uniqR = extractConstants(constantsR);
Set<Long> init = new TreeSet<>();
init.addAll(uniqL);
uniqR.removeAll(plugin.getExcludedOffsets());
Set<Long> common = new HashSet<>(uniqL);
common.retainAll(uniqR);
// Remove what's in common
uniqL.removeAll(common);
uniqR.removeAll(common);
if (uniqL.isEmpty() || uniqR.isEmpty()) {
if (init.contains(offset)) {
upcount(offset, offset);
}
return;
}
// Use obvious matches
if (uniqL.size() == 1 && uniqL.contains(offset) && uniqR.size() == 1) {
upcount(offset, uniqR.iterator().next());
}
}
private void upcount(Long offsetL, Long offsetR) {
results.computeIfAbsent(offsetL, _ -> new HashMap<>())
.merge(offsetR, 1.0f, Float::sum);
}
private Set<Long> extractConstants(Set<Varnode> constants) {
Set<Long> uniq = new TreeSet<>();
for (Varnode vn : constants) {
PcodeOp descendent = vn.getLoneDescend();
if (descendent != null) {
int opcode = descendent.getOpcode();
if (opcode == PcodeOp.INDIRECT) {
continue;
}
if (opcode == PcodeOp.PTRADD) {
Varnode input1 = descendent.getInput(1);
Varnode input2 = descendent.getInput(2);
if (input1.isConstant() && input2.isConstant()) {
uniq.add(Math.abs(input1.getOffset() * input2.getOffset()));
continue;
}
}
}
long vnOffset = vn.getOffset();
if (vnOffset == 0 || vnOffset == 1) {
continue;
}
vnOffset = Math.abs(vnOffset);
if (vnOffset > maxOffset) {
continue;
}
uniq.add(vnOffset);
}
return uniq;
}
private void collectConstantsByOperand(PcodeOp op, Set<PcodeOp> visitedOps,
Set<Varnode> constants, int depth) {
if (depth < 0 || !visitedOps.add(op)) {
return;
}
int nextDepth = depth - 1;
int numInputs = op.getNumInputs();
for (int i = 0; i < numInputs; i++) {
Varnode input = op.getInput(i);
if (input != null) {
collectConstantsByVarnode(input, visitedOps, constants, nextDepth);
}
}
}
private void collectConstantsByVarnode(Varnode vn, Set<PcodeOp> visitedOps,
Set<Varnode> constants, int depth) {
if (depth < 0) {
return;
}
if (vn.isConstant()) {
constants.add(vn);
}
PcodeOp def = vn.getDef();
if (def != null) {
collectConstantsByOperand(def, visitedOps, constants, depth);
}
}
private void buildStructure() {
Structure original = plugin.getTargetDataType();
if (original == null) {
console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,
"Error: Target data type is missing.");
return;
}
List<PinningMatch> matchHistory = new ArrayList<>();
long max = -1L;
for (Entry<Long, Map<Long, Float>> entry : results.entrySet()) {
Long currentOffset = entry.getKey();
DataTypeComponent c = original.getComponentAt(currentOffset.intValue());
int srcOffset = entry.getKey().intValue();
Map<Long, Float> targetMap = entry.getValue();
for (Entry<Long, Float> targetEntry : targetMap.entrySet()) {
Long targetOffset = targetEntry.getKey();
if (targetOffset > max) {
max = targetOffset + c.getLength();
}
Float weight = targetEntry.getValue();
if (weight != null) {
matchHistory.add(new PinningMatch(weight, srcOffset, targetOffset.intValue()));
}
}
}
matchHistory.sort((a, b) -> Float.compare(b.weight, a.weight));
try (Transaction _ = targetProgram.openTransaction("Build structure")) {
ProgramBasedDataTypeManager dtm = targetProgram.getDataTypeManager();
Structure structure =
new StructureDataType("RECOVERED_" + original.getName(), (int) max, dtm);
Set<Integer> placed = new HashSet<>();
Set<Integer> occupiedBytes = new HashSet<>();
for (PinningMatch match : matchHistory) {
if (placed.contains(match.sourceOffset)) {
continue;
}
DataTypeComponent origComp = original.getComponentAt(match.sourceOffset);
DataTypeComponent targetComp = structure.getComponentAt(match.targetOffset);
DataType dataType = origComp.getDataType();
DataType targetDataType = dtm.getDataType(dataType.getDataTypePath());
if (targetComp != null && targetComp.isUndefined()) {
if (targetDataType == null) {
// Try to resolve or replicate the type context locally if path missing
targetDataType =
dtm.resolve(dataType, DataTypeConflictHandler.DEFAULT_HANDLER);
}
if (targetDataType == null) {
targetDataType = DataType.DEFAULT;
}
int componentLength = targetDataType.getLength();
if (componentLength <= 0) {
componentLength = origComp.getLength();
}
// Check for overlapping byte targets (possibly overkill,, but...)
boolean conflict = false;
for (int i = 0; i < componentLength; i++) {
if (occupiedBytes.contains(match.targetOffset + i)) {
conflict = true;
break;
}
}
if (conflict) {
console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,
"Overlapping target bounds for " + match + " : " +
origComp + " using Byte");
targetDataType = ByteDataType.dataType;
componentLength = targetDataType.getLength();
}
try {
structure.replaceAtOffset(match.targetOffset, targetDataType,
targetDataType.getLength(), origComp.getFieldName(),
origComp.getComment());
placed.add(match.sourceOffset);
for (int i = 0; i < componentLength; i++) {
occupiedBytes.add(match.targetOffset + i);
}
}
catch (IllegalArgumentException iae) {
console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,
iae.getMessage());
}
}
}
console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,
"Placed: " + placed.size() + " in " + structure.getName());
DataTypeComponent[] definedComponents = structure.getDefinedComponents();
DataTypeComponent last = definedComponents[definedComponents.length - 1];
structure.setLength(last.getEndOffset() + 1);
dtm.addDataType(structure, DataTypeConflictHandler.DEFAULT_HANDLER);
}
}
}

320
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java Normal file
View File

@@ -0,0 +1,320 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.gui.structs;
import java.util.*;
import org.jgrapht.Graph;
import org.jgrapht.alg.interfaces.MatchingAlgorithm.Matching;
import org.jgrapht.alg.matching.MaximumWeightBipartiteMatching;
import org.jgrapht.graph.DefaultWeightedEdge;
import org.jgrapht.graph.SimpleWeightedGraph;
import db.Transaction;
import generic.lsh.vector.*;
import ghidra.app.decompiler.DecompileException;
import ghidra.app.services.ConsoleService;
import ghidra.features.bsim.gui.structs.StructureRecoveryPlugin.FunctionMatchOption;
import ghidra.features.bsim.query.*;
import ghidra.features.bsim.query.client.Configuration;
import ghidra.features.bsim.query.description.FunctionDescription;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.*;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
public class MatchFunctionsTask extends Task {
//functions with self significance below this bound will be skipped
private double SELF_SIGNIFICANCE_BOUND = 15.0;
//bsim database template determining the signature settings
private String TEMPLATE_NAME = "medium_nosize";
//these are analogous to the bounds in a bsim query
private double MATCH_SIMILARITY_LOWER_BOUND = 0.2;
//decrease this if you only want to see matches that aren't exact
//for instance, when looking for changes between two versions of a program
private double MATCH_SIMILARITY_UPPER_BOUND = 1.0;
private double MATCH_CONFIDENCE_LOWER_BOUND = 0.0;
// We'll consider solo matches or matches above this confidence
private double MIN_CONFIDENCE = 20.0;
private StructureRecoveryPlugin plugin;
private ConsoleService console;
private Program currentProgram;
private Program targetProgram;
private BookmarkManager srcBookmarks;
private BookmarkManager tgtBookmarks;
private Set<Function> functionsToMatch = new HashSet<>();
Map<Function, Set<Function>> functionMap = new HashMap<>();
public final static String TAG = "Function Match";
public MatchFunctionsTask(StructureRecoveryPlugin plugin, Program target) {
super("Match Functions", true, false, false);
this.plugin = plugin;
this.targetProgram = target;
this.functionsToMatch = plugin.getFunctionsToMatch();
this.console = plugin.getConsole();
this.currentProgram = plugin.getCurrentProgram();
this.SELF_SIGNIFICANCE_BOUND = plugin.getSelfSiginificanceBound();
this.MATCH_SIMILARITY_LOWER_BOUND = plugin.getMatchSimilarityLowerBound();
this.MATCH_SIMILARITY_UPPER_BOUND = plugin.getMatchSimilarityUpperBound();
this.MATCH_CONFIDENCE_LOWER_BOUND = plugin.getMatchConfidenceLowerBound();
this.MIN_CONFIDENCE = plugin.getMinConfidence();
this.srcBookmarks = currentProgram.getBookmarkManager();
this.tgtBookmarks = targetProgram.getBookmarkManager();
}
@Override
public void run(TaskMonitor monitor) {
String taskName = getTaskTitle();
try {
Thread.currentThread().setName(taskName);
console.addMessage(taskName, "Running...");
matchFunctions(monitor);
console.addMessage(taskName, "Finished!");
}
catch (CancelledException e) {
console.addMessage(taskName, "Cancelled by user.");
}
catch (Exception e) {
if (!monitor.isCancelled()) {
Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName +
"\n" + e.getClass().getName() + ": " + e.getMessage(), e);
console.addErrorMessage("", "Error running task: " + taskName);
console.addException(taskName, e);
}
}
}
private void matchFunctions(TaskMonitor monitor)
throws CancelledException, LSHException, DecompileException {
Iterator<Function> functionIter = targetProgram.getFunctionManager().getFunctions(true);
GenSignatures targetSigs =
generateSignatures(targetProgram, functionIter, getVectorFactory(), monitor);
List<LocalBSimMatch> matchResults =
getMatchesTwoPrograms(functionsToMatch, targetSigs, monitor);
FunctionMatchOption matchType = plugin.getOptionSoloMatchesOnly();
if (matchType != FunctionMatchOption.MAX_WEIGHT) {
defineMapFromSoloMatches(matchResults, matchType, monitor);
}
else {
defineMap(matchResults, monitor);
}
}
private void defineMapFromSoloMatches(List<LocalBSimMatch> matchResults,
FunctionMatchOption matchType, TaskMonitor monitor) {
boolean addBookmarks = plugin.getOptionFnMatchBookmarks();
try (Transaction _ = currentProgram.openTransaction("add");
Transaction _ = targetProgram.openTransaction("add")) {
Set<String> processed = new HashSet<>();
for (LocalBSimMatch match : matchResults) {
Function left = match.sourceFunc();
Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());
Function right = match.targetFunc();
if (right != null && !matchType.equals(FunctionMatchOption.MAX_WEIGHT)) {
set.add(right);
Double sig = match.significance();
if (addBookmarks && sig >= MIN_CONFIDENCE) {
addBookmarks(left, right, processed);
}
}
}
for (Function left : functionMap.keySet()) {
Set<Function> set = functionMap.get(left);
if (set.size() == 1) {
Function right = set.iterator().next();
if (addBookmarks) {
addBookmarks(left, right, processed);
}
}
}
}
plugin.setFunctionMap(functionMap);
}
private void defineMap(List<LocalBSimMatch> matchResults, TaskMonitor monitor)
throws CancelledException {
Set<String> setLeft = new HashSet<>();
Set<String> setRight = new HashSet<>();
Graph<String, DefaultWeightedEdge> graph =
new SimpleWeightedGraph<>(DefaultWeightedEdge.class);
monitor.setMessage("Computing local BSim results...");
monitor.setMaximum(matchResults.size());
for (LocalBSimMatch match : matchResults) {
monitor.checkCancelled();
Function left = match.sourceFunc();
Function right = match.targetFunc();
if (right != null) {
Double sig = match.significance();
String leftID = "L" + Long.toString(left.getID());
String rightID = "R" + Long.toString(right.getID());
setLeft.add(leftID);
graph.addVertex(leftID);
setRight.add(rightID);
graph.addVertex(rightID);
addWeightedEdge(graph, leftID, rightID, sig);
}
}
MaximumWeightBipartiteMatching<String, DefaultWeightedEdge> alg =
new MaximumWeightBipartiteMatching<>(graph, setLeft, setRight);
boolean addBookmarks = plugin.getOptionFnMatchBookmarks();
try (Transaction _ = currentProgram.openTransaction("add");
Transaction _ = targetProgram.openTransaction("add")) {
FunctionManager sourceManager = currentProgram.getFunctionManager();
FunctionManager targetManager = targetProgram.getFunctionManager();
monitor.setMessage("Computing match...");
Matching<String, DefaultWeightedEdge> matching = alg.getMatching();
Set<String> processedTags = new HashSet<>();
monitor.setMessage("Computing map...");
monitor.setMaximum(matching.getEdges().size());
for (DefaultWeightedEdge edge : matching.getEdges()) {
monitor.checkCancelled();
String edgeSource = graph.getEdgeSource(edge).substring(1);
Function left = sourceManager.getFunction(Long.parseLong(edgeSource));
String edgeTarget = graph.getEdgeTarget(edge).substring(1);
Function right = targetManager.getFunction(Long.parseLong(edgeTarget));
Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());
set.add(right);
if (addBookmarks) {
addBookmarks(left, right, processedTags);
}
}
}
plugin.setFunctionMap(functionMap);
}
private static void addWeightedEdge(Graph<String, DefaultWeightedEdge> graph, String source,
String target, double weight) {
DefaultWeightedEdge edge = graph.addEdge(source, target);
if (edge != null) {
graph.setEdgeWeight(edge, weight);
}
}
private void addBookmarks(Function left, Function right, Set<String> processed) {
String key = left.getID() + "->" + right.getID();
if (processed.add(key)) {
srcBookmarks.setBookmark(left.getEntryPoint(), TAG,
Long.toString(left.getID()), Long.toString(right.getID()));
tgtBookmarks.setBookmark(right.getEntryPoint(), TAG,
Long.toString(left.getID()), Long.toString(right.getID()));
}
}
public record LocalBSimMatch(Function sourceFunc, Function targetFunc, double sim,
double significance) {}
private List<LocalBSimMatch> getMatchesTwoPrograms(Set<Function> srcFuncs,
GenSignatures targetSigs, TaskMonitor monitor)
throws LSHException, DecompileException, CancelledException {
LSHVectorFactory vectorFactory = getVectorFactory();
GenSignatures srcSigs =
generateSignatures(currentProgram, srcFuncs.iterator(), vectorFactory, monitor);
Iterator<FunctionDescription> sourceDescripts =
srcSigs.getDescriptionManager().listAllFunctions();
VectorCompare vecCompare = new VectorCompare();
List<LocalBSimMatch> bsimMatches = new ArrayList<>();
List<FunctionDescription> targetList = new ArrayList<>();
Iterator<FunctionDescription> targetDescriptsIter =
targetSigs.getDescriptionManager().listAllFunctions();
while (targetDescriptsIter.hasNext()) {
monitor.checkCancelled();
targetList.add(targetDescriptsIter.next());
}
while (sourceDescripts.hasNext()) {
monitor.checkCancelled();
FunctionDescription srcDesc = sourceDescripts.next();
//skip if self-significance too small
LSHVector srcVector = srcDesc.getSignatureRecord().getLSHVector();
if (vectorFactory.getSelfSignificance(srcVector) <= SELF_SIGNIFICANCE_BOUND) {
continue;
}
Function srcFunc = getFunction(currentProgram, srcDesc.getAddress());
for (FunctionDescription targetDesc : targetList) {
//skip if self-significance too small
LSHVector targetVector = targetDesc.getSignatureRecord().getLSHVector();
if (vectorFactory.getSelfSignificance(targetVector) <= SELF_SIGNIFICANCE_BOUND) {
continue;
}
double sim = srcVector.compare(targetVector, vecCompare);
double sig = vectorFactory.calculateSignificance(vecCompare);
if (sig >= MATCH_CONFIDENCE_LOWER_BOUND && MATCH_SIMILARITY_LOWER_BOUND <= sim &&
sim <= MATCH_SIMILARITY_UPPER_BOUND) {
Function targetFunc = getFunction(targetProgram, targetDesc.getAddress());
if (targetFunc != null) {
bsimMatches.add(new LocalBSimMatch(srcFunc, targetFunc, sim, sig));
}
else {
Msg.warn(this, "Function not found for " + targetDesc.getAddress());
}
}
monitor.checkCancelled();
}
}
return bsimMatches;
}
private Function getFunction(Program program, long offset) {
Address addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(offset);
return program.getFunctionManager().getFunctionAt(addr);
}
private LSHVectorFactory getVectorFactory() throws LSHException {
LSHVectorFactory vectorFactory = FunctionDatabase.generateLSHVectorFactory();
Configuration config = FunctionDatabase.loadConfigurationTemplate(TEMPLATE_NAME);
vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings);
return vectorFactory;
}
private GenSignatures generateSignatures(Program program, Iterator<Function> funcs,
LSHVectorFactory vectorFactory, TaskMonitor monitor)
throws LSHException, DecompileException {
GenSignatures gensig = null;
try {
int count = program.getFunctionManager().getFunctionCount();
gensig = new GenSignatures(false);
gensig.setVectorFactory(vectorFactory);
gensig.openProgram(program, null, null, null, null, null);
gensig.scanFunctions(funcs, count, monitor);
return gensig;
}
finally {
gensig.dispose();
}
}
}

115
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java Normal file
View File

@@ -0,0 +1,115 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.gui.structs;
import java.util.*;
import ghidra.app.plugin.core.navigation.locationreferences.LocationReference;
import ghidra.app.plugin.core.navigation.locationreferences.ReferenceUtils;
import ghidra.app.services.ConsoleService;
import ghidra.app.services.FieldMatcher;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.Structure;
import ghidra.program.model.listing.*;
import ghidra.util.Msg;
import ghidra.util.datastruct.SetAccumulator;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.Task;
import ghidra.util.task.TaskMonitor;
public class RetrieveUsesTask extends Task {
private StructureRecoveryPlugin plugin;
private Structure targetDataType;
private ConsoleService console;
private Program currentProgram;
public final static String TAG = "Structure Use";
public RetrieveUsesTask(StructureRecoveryPlugin plugin) {
super("Retrieve Structure Uses", true, false, false);
this.plugin = plugin;
this.targetDataType = plugin.getTargetDataType();
this.console = plugin.getConsole();
this.currentProgram = plugin.getCurrentProgram();
}
@Override
public void run(TaskMonitor monitor) {
String taskName = getTaskTitle();
Thread.currentThread().setName(taskName);
try {
console.addMessage(taskName, "Running...");
retrieveReferences(monitor);
console.addMessage(taskName, "Finished!");
}
catch (CancelledException e) {
console.addMessage(taskName, "Cancelled by user.");
}
catch (Exception e) {
if (!monitor.isCancelled()) {
Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName +
"\n" + e.getClass().getName() + ": " + e.getMessage(), e);
console.addErrorMessage("", "Error running task: " + taskName);
console.addException(taskName, e);
}
}
}
protected void retrieveReferences(TaskMonitor monitor) throws CancelledException {
FieldMatcher fieldMatcher = new FieldMatcher(targetDataType);
SetAccumulator<LocationReference> accumulator = new SetAccumulator<>();
ReferenceUtils.findDataTypeFieldReferences(accumulator, fieldMatcher, currentProgram, true,
monitor);
Set<Function> toMatch = new HashSet<>();
Map<String, Set<Function>> byField = new HashMap<>();
Map<String, AddressSet> toAddressSet = new HashMap<>();
boolean useBookmarks = plugin.getOptionStructUseBookmarks();
currentProgram.withTransaction("Add Structure Use Bookmarks", () -> {
BookmarkManager bookmarkManager = currentProgram.getBookmarkManager();
FunctionManager functionManager = currentProgram.getFunctionManager();
String typeName = targetDataType.getName();
for (LocationReference lr : accumulator.get()) {
String fieldName = lr.getFieldName();
if (fieldName == null) {
continue;
}
Address locationOfUse = lr.getLocationOfUse();
Function f = functionManager.getFunctionContaining(locationOfUse);
if (f == null) {
continue;
}
toAddressSet.computeIfAbsent(fieldName, _ -> new AddressSet())
.add(locationOfUse);
byField.computeIfAbsent(fieldName, _ -> new HashSet<Function>()).add(f);
toMatch.add(f);
if (useBookmarks) {
bookmarkManager.setBookmark(locationOfUse, TAG,
"USES_" + typeName, fieldName);
}
}
});
plugin.setFunctionsToMatch(toMatch);
plugin.setFunctionsByField(byField);
plugin.setAddressesByField(toAddressSet);
}
}

686
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java Normal file
View File

@@ -0,0 +1,686 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.gui.structs;
import static ghidra.framework.main.DataTreeDialogType.*;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
import db.Transaction;
import docking.DockingWindowManager;
import docking.action.DockingAction;
import docking.action.MenuData;
import docking.action.builder.ActionBuilder;
import ghidra.app.CorePluginPackage;
import ghidra.app.events.*;
import ghidra.app.merge.structures.StructureMergeDialog;
import ghidra.app.plugin.PluginCategoryNames;
import ghidra.app.plugin.ProgramPlugin;
import ghidra.app.services.ConsoleService;
import ghidra.app.util.datatype.DataTypeSelectionDialog;
import ghidra.framework.main.DataTreeDialog;
import ghidra.framework.model.DomainFile;
import ghidra.framework.model.DomainFileFilter;
import ghidra.framework.options.*;
import ghidra.framework.plugintool.*;
import ghidra.framework.plugintool.util.PluginStatus;
import ghidra.program.model.address.*;
import ghidra.program.model.data.*;
import ghidra.program.model.listing.*;
import ghidra.program.util.ProgramLocation;
import ghidra.util.*;
import ghidra.util.bean.opteditor.OptionsVetoException;
import ghidra.util.data.DataTypeParser.AllowedDataTypes;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.VersionException;
import ghidra.util.task.TaskMonitor;
/**
* Plugin for recovering structure layouts across programs.
*/
//@formatter:off
@PluginInfo(
status = PluginStatus.UNSTABLE,
packageName = CorePluginPackage.NAME,
category = PluginCategoryNames.ANALYSIS,
shortDescription = "StructureRecovery",
description = "Plugin for recovering structure layouts across programs",
servicesRequired = {
ConsoleService.class
},
eventsConsumed = {
ProgramActivatedPluginEvent.class, ProgramOpenedPluginEvent.class,
ProgramLocationPluginEvent.class, ProgramSelectionPluginEvent.class,
ProgramClosedPluginEvent.class
})
//@formatter:on
public class StructureRecoveryPlugin extends ProgramPlugin implements OptionsChangeListener {
static final String TAG_STRUCTURE_USE = "Tag structure use";
static final String MATCH_FUNCTIONS = "Match functions";
static final String REGENERATE_STRUCT = "Generate structure";
static final String COMPARE_STRUCTURES = "Compare new structure";
private static final String FUNCTIONS_MAP_BKMKS = "Function matches: map to bookmarks";
private static final String FUNCTIONS_BKMKS_MAP = "Function matches: bookmarks to map";
private static final String FUNCTIONS_BKMKS_DEL = "Function matches: remove bookmarks";
private static final String USES_MAP_BKMKS = "Structure uses: map to bookmarks";
private static final String USES_BKMKS_MAP = "Structure uses: bookmarks to map";
private static final String USES_BKMKS_DEL = "Structure uses: remove bookmarks";
public final static String HELP_LOCATION = "StructureRecovery";
private static final String DEFAULT_STRUCTURE = "Structure to recover";
private static final String ADD_STRUCTURE_USE_BKMKS = "Add structure-use bookmarks";
private static final String ADD_FUNCTION_MATCH_BKMKS = "Add function-match bookmarks";
private static final String SELF_SIGNIFICANCE_BOUND = "Self-significance bound";
private static final String MATCH_SIMILARITY_LOWER_BOUND = "Match-similarity lower bound";
private static final String MATCH_SIMILARITY_UPPER_BOUND = "Match-similarity upper bound";
private static final String MATCH_CONFIDENCE_LOWER_BOUND = "Match-confidence lower bound";
private static final String MIN_CONFIDENCE = "Min confidence";
private static final String MAX_OFFSET = "Max structure offset";
private static final String EXCLUDED_OFFSETS = "Excluded offsets (CSV hex values)";
private static final String MATCH_ON = "Match on";
public enum FunctionMatchOption {
SOLO_MATCHES, SOLO_OR_MIN_CONFIDENCE, MAX_WEIGHT
}
Program targetProgram;
Map<Function, Set<Function>> functionMap = new HashMap<>();
Map<Function, Float> confidence = new HashMap<>();
private Function currentFunction;
private ConsoleService consoleService;
private Structure targetDataType;
private Map<String, Long> offsets = new HashMap<>();
private Map<Long, String> names = new HashMap<>();
private Set<Function> srcFunctionsToMatch = new HashSet<>();
private Map<String, Set<Function>> srcFunctionsByField = new HashMap<>();
private Map<String, AddressSet> srcAddressesByField = new HashMap<>();
private ToolOptions options;
private DockingAction tagAction;
private DockingAction regenerateAction;
private DockingAction compareAction;
public StructureRecoveryPlugin(PluginTool tool) {
super(tool);
createActions();
getOptions();
}
public Function getCurrentFunction() {
return currentFunction;
}
@Override
protected void programActivated(Program program) {
currentProgram = program;
}
@Override
protected void programClosed(Program program) {
currentProgram = null;
}
private void createActions() {
tagAction = new ActionBuilder(TAG_STRUCTURE_USE, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
if (checkDataType()) {
tool.execute(new RetrieveUsesTask(StructureRecoveryPlugin.this));
}
})
.menuGroup("A", "a")
.menuPath("Tools", "Structure recovery", TAG_STRUCTURE_USE)
.buildAndInstall(tool);
new ActionBuilder(MATCH_FUNCTIONS, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsToMatch.isEmpty())
.onAction(_ -> {
tool.execute(
new MatchFunctionsTask(StructureRecoveryPlugin.this, getTargetProgram()));
})
.menuGroup("A", "b")
.menuPath("Tools", "Structure recovery", MATCH_FUNCTIONS)
.buildAndInstall(tool);
regenerateAction = new ActionBuilder(REGENERATE_STRUCT, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty() &&
targetProgram != currentProgram)
.onAction(_ -> {
if (checkDataType()) {
tool.execute(new ComparePinningsTask(StructureRecoveryPlugin.this));
}
})
.menuGroup("A", "c")
.menuPath("Tools", "Structure recovery", REGENERATE_STRUCT)
.buildAndInstall(tool);
compareAction = new ActionBuilder(COMPARE_STRUCTURES, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
compareStructures();
})
.menuGroup("A", "d")
.menuPath("Tools", "Structure recovery", COMPARE_STRUCTURES)
.buildAndInstall(tool);
new ActionBuilder(USES_BKMKS_MAP, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
if (checkDataType()) {
getConsole().addMessage(USES_BKMKS_MAP, "Running...");
regenerateUsesMapFromBookmarks();
getConsole().addMessage(USES_BKMKS_MAP, "Finished!");
}
})
.menuGroup("B", "a")
.menuPath("Tools", "Structure recovery", USES_BKMKS_MAP)
.buildAndInstall(tool);
new ActionBuilder(USES_MAP_BKMKS, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsByField.isEmpty())
.onAction(_ -> {
getConsole().addMessage(USES_MAP_BKMKS, "Running...");
generateUsesMapBookmarks();
getConsole().addMessage(USES_MAP_BKMKS, "Finished!");
})
.menuGroup("B", "b")
.menuPath("Tools", "Structure recovery", USES_MAP_BKMKS)
.buildAndInstall(tool);
new ActionBuilder(FUNCTIONS_BKMKS_MAP, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Running...");
regenerateFunctionMapFromBookmarks();
getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Finished!");
})
.menuGroup("B", "c")
.menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_MAP)
.buildAndInstall(tool);
new ActionBuilder(FUNCTIONS_MAP_BKMKS, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty())
.onAction(_ -> {
getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Running...");
generateFunctionMapBookmarks();
getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Finished!");
})
.menuGroup("B", "d")
.menuPath("Tools", "Structure recovery", FUNCTIONS_MAP_BKMKS)
.buildAndInstall(tool);
new ActionBuilder(USES_BKMKS_DEL, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
wipeUsesMapBookmarks();
})
.menuGroup("B", "e")
.menuPath("Tools", "Structure recovery", USES_BKMKS_DEL)
.buildAndInstall(tool);
new ActionBuilder(FUNCTIONS_BKMKS_DEL, HELP_LOCATION)
.enabledWhen(_ -> getCurrentProgram() != null)
.onAction(_ -> {
wipeFunctionMapBookmarks();
})
.menuGroup("B", "f")
.menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_DEL)
.buildAndInstall(tool);
}
@Override
public void optionsChanged(ToolOptions options, String optionName, Object oldValue,
Object newValue) throws OptionsVetoException {
if (optionName.equals(DEFAULT_STRUCTURE)) {
checkDataType();
boolean valid = targetDataType != null;
String tagCmd = !valid ? TAG_STRUCTURE_USE : "Tag " + targetDataType.getName() + " use";
String regenCmd = !valid ? REGENERATE_STRUCT
: "Generate " + targetDataType.getName();
if (targetProgram != null) {
regenCmd += " (" + targetProgram + ")";
}
String compCmd = !valid ? COMPARE_STRUCTURES
: "Compare " + targetDataType.getName() + " structures";
tagAction.setMenuBarData(
new MenuData(new String[] { "Tools", "Structure recovery", tagCmd }));
regenerateAction.setMenuBarData(new MenuData(
new String[] { "Tools", "Structure recovery", regenCmd }));
compareAction.setMenuBarData(new MenuData(
new String[] { "Tools", "Structure recovery", compCmd }));
}
}
@Override
public Program getCurrentProgram() {
return currentProgram;
}
@Override
public void close() {
if (targetProgram != null) {
targetProgram.release(this);
}
}
@Override
public void processEvent(PluginEvent event) {
super.processEvent(event);
//Msg.info(this, "TaintPlugin -> processEvent: " + event.toString() );
if (event instanceof ProgramClosedPluginEvent) {
if (currentProgram != null) {
currentProgram.release(this);
}
return;
}
if (event instanceof ProgramActivatedPluginEvent) {
currentProgram = ((ProgramActivatedPluginEvent) event).getActiveProgram();
}
else if (event instanceof ProgramLocationPluginEvent) {
ProgramLocation location = ((ProgramLocationPluginEvent) event).getLocation();
Address address = location.getAddress();
if (address.isExternalAddress()) {
// ignore external functions when it comes to taint.
return;
}
if (currentProgram != null) {
// The user loaded a program for analysis.
Listing listing = currentProgram.getListing();
Function f = listing.getFunctionContaining(address);
// We are in function f
if (currentFunction == null || !currentFunction.equals(f)) {
currentFunction = f;
}
}
}
}
private boolean checkDataType() {
if (targetDataType != null) {
return true;
}
if (currentProgram == null) {
Msg.showWarn(this, null, "No Program",
"Please open a program before selecting a structure.");
return false;
}
DataTypeManager dtm = currentProgram.getDataTypeManager();
String structureName = getDefaultStructureName();
DataType dataType = null;
if (!structureName.equals("")) {
dataType = dtm.getDataType(structureName);
if (dataType == null) {
List<DataType> list = new ArrayList<>();
dtm.findDataTypes(structureName, list);
if (list.isEmpty()) {
Msg.error(this, dataType + " in ToolOptions is not valid.");
return false;
}
dataType = list.get(0);
}
}
else {
DataTypeSelectionDialog selectionDialog =
new DataTypeSelectionDialog(tool, dtm, -1, AllowedDataTypes.FIXED_LENGTH);
tool.showDialog(selectionDialog);
dataType = selectionDialog.getUserChosenDataType();
if (dataType == null) {
return false;
}
options.setString(DEFAULT_STRUCTURE, dataType.getPathName());
}
if (dataType instanceof Structure struct) {
targetDataType = struct;
offsets.clear();
names.clear();
for (DataTypeComponent dtc : targetDataType.getComponents()) {
if (dtc.getFieldName() != null && dtc.getOffset() > 0) {
offsets.put(dtc.getFieldName(), (long) dtc.getOffset());
names.put((long) dtc.getOffset(), dtc.getFieldName());
}
}
return true;
}
return false;
}
public Program loadTargetProgram() throws VersionException, IOException, CancelledException {
DomainFileFilter filter = df -> Program.class.isAssignableFrom(df.getDomainObjectClass());
DataTreeDialog dtd = new DataTreeDialog(null, "Target Program", OPEN, filter);
dtd.show();
if (dtd.wasCancelled()) {
return null;
}
DomainFile df = dtd.getDomainFile();
return (Program) df.getDomainObject(this, true, false, TaskMonitor.DUMMY);
}
private void generateFunctionMapBookmarks() {
BookmarkManager srcBookmarks = currentProgram.getBookmarkManager();
BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager();
try (Transaction _ = currentProgram.openTransaction("add")) {
try (Transaction _ = targetProgram.openTransaction("add")) {
for (Entry<Function, Set<Function>> entry : functionMap.entrySet()) {
Function left = entry.getKey();
for (Function right : entry.getValue()) {
srcBookmarks.setBookmark(left.getEntryPoint(), MatchFunctionsTask.TAG,
Long.toString(left.getID()), Long.toString(right.getID()));
tgtBookmarks.setBookmark(right.getEntryPoint(), MatchFunctionsTask.TAG,
Long.toString(left.getID()), Long.toString(right.getID()));
}
}
}
}
}
private void regenerateFunctionMapFromBookmarks() {
getTargetProgram();
FunctionManager srcMgr = currentProgram.getFunctionManager();
FunctionManager tgtMgr = targetProgram.getFunctionManager();
boolean adds = false;
BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();
Iterator<Bookmark> iterator = bkmkMgr.getBookmarksIterator(MatchFunctionsTask.TAG);
while (iterator.hasNext()) {
Bookmark mark = iterator.next();
try {
Function left = srcMgr.getFunction(Long.parseLong(mark.getCategory()));
if (left == null) {
continue;
}
Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());
String tgtId = mark.getComment();
Function right = tgtMgr.getFunction(Long.parseLong(tgtId));
if (right == null) {
continue;
}
set.add(right);
adds = true;
}
catch (NumberFormatException e) {
// Skip
}
}
if (!adds) {
getConsole().addErrorMessage(FUNCTIONS_BKMKS_MAP,
"Map empty - check function id bookmarks");
}
}
private void wipeFunctionMapBookmarks() {
try (Transaction _ = currentProgram.openTransaction("wipeCurrent")) {
getTargetProgram();
BookmarkManager srcBookmarks = currentProgram.getBookmarkManager();
srcBookmarks.removeBookmarks(MatchFunctionsTask.TAG);
}
getTargetProgram();
try (Transaction _ = targetProgram.openTransaction("wipeTarget")) {
getTargetProgram();
BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager();
tgtBookmarks.removeBookmarks(MatchFunctionsTask.TAG);
}
}
private void generateUsesMapBookmarks() {
BookmarkManager bookmarkManager = currentProgram.getBookmarkManager();
try (Transaction _ = currentProgram.openTransaction("add")) {
for (Entry<String, Set<Function>> entry : srcFunctionsByField.entrySet()) {
String fname = entry.getKey();
AddressSet addressSet = srcAddressesByField.get(fname);
if (addressSet == null) {
continue;
}
Iterator<Function> iterator = entry.getValue().iterator();
while (iterator.hasNext()) {
Function f = iterator.next();
AddressSet intersect = addressSet.intersect(f.getBody());
if (intersect == null) {
continue;
}
// Add a bookmark for each intersecting address range's minimum address so
// it can be parsed back into a single address when rebuilding the map.
for (AddressRange range : intersect.getAddressRanges()) {
bookmarkManager.setBookmark(range.getMinAddress(), RetrieveUsesTask.TAG,
"USES_" + targetDataType.getName(), fname);
}
}
}
}
}
private void regenerateUsesMapFromBookmarks() {
FunctionManager srcMgr = currentProgram.getFunctionManager();
BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();
Iterator<Bookmark> iterator = bkmkMgr.getBookmarksIterator(RetrieveUsesTask.TAG);
while (iterator.hasNext()) {
Bookmark mark = iterator.next();
AddressSet aset =
srcAddressesByField.computeIfAbsent(mark.getComment(), _ -> new AddressSet());
Address address = mark.getAddress();
if (address != null) {
aset.add(address);
}
Set<Function> set =
srcFunctionsByField.computeIfAbsent(mark.getComment(), _ -> new HashSet<>());
Function f = srcMgr.getFunctionContaining(address);
if (f != null) {
set.add(f);
srcFunctionsToMatch.add(f);
}
}
if (srcFunctionsToMatch.isEmpty()) {
getConsole().addErrorMessage(USES_BKMKS_MAP, "Map empty - check USES bookmarks");
}
}
private void wipeUsesMapBookmarks() {
try (Transaction _ = currentProgram.openTransaction("wipe")) {
BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();
bkmkMgr.removeBookmarks(RetrieveUsesTask.TAG);
}
}
private void compareStructures() {
checkDataType();
getTargetProgram();
if (targetProgram == null) {
consoleService.addErrorMessage("Compare Structures", "Target program is null.");
return;
}
DataTypeManager dtm = targetProgram.getDataTypeManager();
String structureName = "/RECOVERED_" + targetDataType.getName();
DataType dataType = dtm.getDataType(structureName);
if (dataType == null) {
consoleService.addErrorMessage("Compare Structures", structureName + " not found.");
return;
}
if (dataType instanceof Structure recoveredStructure) {
StructureMergeDialog dialog =
new StructureMergeDialog("Compare Structures", recoveredStructure, targetDataType,
_ -> {
//IGNORE
});
DockingWindowManager.showDialog(dialog);
}
}
public Map<String, Long> getOffsets() {
return offsets;
}
public Map<Long, String> getFieldNames() {
return names;
}
public Map<Function, Set<Function>> getFunctionMap() {
return functionMap;
}
public void setFunctionMap(Map<Function, Set<Function>> map) {
this.functionMap = map;
}
public Map<String, Set<Function>> getFunctionsByField() {
return srcFunctionsByField;
}
public void setFunctionsByField(Map<String, Set<Function>> functionsByField) {
this.srcFunctionsByField = functionsByField;
}
public Map<String, AddressSet> getAddressesByField() {
return srcAddressesByField;
}
public void setAddressesByField(Map<String, AddressSet> toAddressSet) {
this.srcAddressesByField = toAddressSet;
}
public Set<Function> getFunctionsToMatch() {
return srcFunctionsToMatch;
}
public void setFunctionsToMatch(Set<Function> srcFunctionsToMatch) {
this.srcFunctionsToMatch = srcFunctionsToMatch;
}
public Structure getTargetDataType() {
return targetDataType;
}
public ConsoleService getConsole() {
if (consoleService == null) {
consoleService = tool.getService(ConsoleService.class);
}
return consoleService;
}
public Program getTargetProgram() {
if (targetProgram == null) {
Swing.runNow(() -> {
try {
targetProgram = loadTargetProgram();
}
catch (VersionException | CancelledException | IOException e) {
getConsole().addMessage("Load target", e.getMessage());
}
});
}
return targetProgram;
}
private void getOptions() {
options = tool.getOptions("Structure Recovery");
HelpLocation help = new HelpLocation(getName(), "Options");
options.registerOption(DEFAULT_STRUCTURE, OptionType.STRING_TYPE, "", help,
DEFAULT_STRUCTURE);
options.registerOption(ADD_STRUCTURE_USE_BKMKS, OptionType.BOOLEAN_TYPE, true, help,
"Generate structure-use function bookmarks by default");
options.registerOption(ADD_FUNCTION_MATCH_BKMKS, OptionType.BOOLEAN_TYPE, true, help,
"Generate function-match function bookmarks by default");
options.registerOption(SELF_SIGNIFICANCE_BOUND, OptionType.DOUBLE_TYPE, 15.0, help,
"Functions with self significance below this bound will be skipped");
options.registerOption(MATCH_SIMILARITY_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.3, help,
"Lower threshold equates to looser matches");
options.registerOption(MATCH_SIMILARITY_UPPER_BOUND, OptionType.DOUBLE_TYPE, 1.0, help,
"Decrease this if you only want to see matches that aren't exact");
options.registerOption(MATCH_CONFIDENCE_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.0, help,
MATCH_CONFIDENCE_LOWER_BOUND);
options.registerOption(MIN_CONFIDENCE, OptionType.DOUBLE_TYPE, 20.0, help,
"Consider matches above this confidence");
options.registerOption(MAX_OFFSET, OptionType.LONG_TYPE, 100000L, help,
"Ignore constants above this value as probable addresses");
options.registerOption(EXCLUDED_OFFSETS, OptionType.STRING_TYPE, "", help,
"Useful for structures using boxed randomized fields");
options.registerOption(MATCH_ON, OptionType.ENUM_TYPE,
FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE, help,
"Match using solo matches, solo+confidence, bipartite pairing");
options.addOptionsChangeListener(this);
}
public String getDefaultStructureName() {
return options.getString(DEFAULT_STRUCTURE, "");
}
public boolean getOptionStructUseBookmarks() {
return options.getBoolean(ADD_STRUCTURE_USE_BKMKS, true);
}
public boolean getOptionFnMatchBookmarks() {
return options.getBoolean(ADD_FUNCTION_MATCH_BKMKS, true);
}
public double getSelfSiginificanceBound() {
return options.getDouble(SELF_SIGNIFICANCE_BOUND, 15.0);
}
public double getMatchSimilarityLowerBound() {
return options.getDouble(MATCH_SIMILARITY_LOWER_BOUND, 0.3);
}
public double getMatchSimilarityUpperBound() {
return options.getDouble(MATCH_SIMILARITY_UPPER_BOUND, 1.0);
}
public double getMatchConfidenceLowerBound() {
return options.getDouble(MATCH_CONFIDENCE_LOWER_BOUND, 0.0);
}
public double getMinConfidence() {
return options.getDouble(MIN_CONFIDENCE, 20.0);
}
public long getMaxOffset() {
return options.getLong(MAX_OFFSET, 100000);
}
public Set<Long> getExcludedOffsets() {
String offstr = options.getString(EXCLUDED_OFFSETS, "");
String[] split = offstr.split(",");
Set<Long> excluded = new HashSet<>();
for (String offset : split) {
if (!offset.equals("")) {
try {
excluded.add(Long.parseLong(offset.trim(), 16));
}
catch (NumberFormatException e) {
// Skip
}
}
}
return excluded;
}
public FunctionMatchOption getOptionSoloMatchesOnly() {
return options.getEnum(MATCH_ON, FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE);
}
}

5
Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java
View File

@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -84,4 +84,5 @@ public interface DataTypeReferenceFinder extends ExtensionPoint {
*/
public void findReferences(Program program, FieldMatcher fieldMatcher,
Consumer<DataTypeReference> callback, TaskMonitor monitor) throws CancelledException;
}

34
Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java
View File

@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -212,4 +212,34 @@ public class DataVertex {
}
return PcodeOp.isCommutative(opc);
}
public PcodeOpAST getOp() {
return op;
}
public VarnodeAST getVarnode() {
return vn;
}
public ArrayList<DataVertex> getSources() {
return sources;
}
/**
* Get the depth at which the pairing occurred
* @return depth of pairing
*/
public int getDepthPaired() {
int size = ngrams.size();
if (size <= 0) {
return 0;
}
DataNGram gramPaired = ngrams.get(size - 1);
for (DataNGram gram : ngrams) {
if (gram.hash == gramPaired.hash) {
return gram.depth;
}
}
return 0;
}
}

4
Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java
View File

@@ -908,4 +908,8 @@ public class Pinning {
}
return result;
}
public Map<DataVertex, DataVertex> getPinMap() {
return pinMap;
}
}