diff --git a/.assets/html/flamegraph.html b/.assets/html/flamegraph.html index 77c1485f..9da318e9 100644 --- a/.assets/html/flamegraph.html +++ b/.assets/html/flamegraph.html @@ -75,9 +75,11 @@ // SPDX-License-Identifier: Apache-2.0 'use strict'; let root, px, pattern; - let level0 = 0, left0 = 0, width0 = 0; + let level0 = 0, left0 = 0, width0 = 0, d = 0; let nav = [], navIndex, matchval; let inverted = false; + const U = undefined; + const maxdiff = -1; const levels = Array(36); for (let h = 0; h < levels.length; h++) { levels[h] = []; @@ -111,10 +113,18 @@ return '#' + (p[0] + ((p[1] * v) << 16 | (p[2] * v) << 8 | (p[3] * v))).toString(16); } + function getDiffColor(diff) { + if (diff === U) return '#ffdd33'; + if (diff === 0) return '#e0e0e0'; + const v = Math.round(128 * (maxdiff - Math.abs(diff)) / maxdiff) + 96; + return diff > 0 ? 'rgb(255,' + v + ',' + v + ')' : 'rgb(' + v + ',' + v + ',255)'; + } + function f(key, level, left, width, inln, c1, int) { levels[level0 = level].push({level, left: left0 += left, width: width0 = width || width0, - color: getColor(palette[key & 7]), title: cpool[key >>> 3], - details: (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '') + color: maxdiff >= 0 ? getDiffColor(d) : getColor(palette[key & 7]), + title: cpool[key >>> 3], + details: (d ? (d > 0 ? ', +' : ', ') + d : '') + (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '') }); } diff --git a/.assets/images/flamegraph_diff.png b/.assets/images/flamegraph_diff.png new file mode 100644 index 00000000..3c9af87a Binary files /dev/null and b/.assets/images/flamegraph_diff.png differ diff --git a/.licenserc.yaml b/.licenserc.yaml index a4c0bec9..d2618e06 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -15,6 +15,7 @@ header: - 'src/jattach' - 'src/res' - '**/MANIFEST.MF' + - 'test/**/*.collapsed' license: content: | Copyright The async-profiler authors diff --git a/docs/ConverterUsage.md b/docs/ConverterUsage.md index 18cc40d6..3d9ecb94 100644 --- a/docs/ConverterUsage.md +++ b/docs/ConverterUsage.md @@ -43,6 +43,8 @@ Conversion options: # otlp: OpenTelemetry profile format. +Differential Flame Graph: + --diff JFR options: --cpu Generate only CPU profile during conversion @@ -120,7 +122,7 @@ jfrconv --cpu foo.jfr for HTML output as HTML is the default format for conversion from JFR. -#### Flame Graph options +### Flame Graph options To add a custom title to the generated Flame Graph, use `--title`, which has the default value `Flame Graph`: @@ -128,9 +130,37 @@ To add a custom title to the generated Flame Graph, use `--title`, which has the jfrconv --cpu foo.jfr foo.html -r --title "Custom Title" ``` -### Other formats +### Differential Flame Graph -`jfrconv` supports converting a JFR file to `collapsed`, `pprof`, `pb.gz` and `heatmap` formats as well. +To find performance regressions, it may be useful to compare current profile +to a previous one that serves as a baseline. Differential Flame Graph +visualizes such a comparsion with a special color scheme: + +- Red color denotes frames with more samples comparing to the baseline (i.e. regression); +- Blue is for frames with less samples; +- Yellow are new frames that were absent in the baseline. + +The more intense the color, the larger the delta. +For each different frame, the delta value is displayed in a tooltip. + +![](/.assets/images/flamegraph_diff.png) + +Differential Flame Graph takes the shape of the current profile: +all frames have exactly the same size as in the normal Flame Graph. +This means, frames that exist only in the base profile will not be visible. +To see such frames, create another differential Flame Graph, +swapping the base and the current input file. + +To create differential Flame Graph, run `jfrconv --diff` with two input files: +basline profile and new profile. Both files can be in JFR, HTML, or collapsed format. +Other converter options work as usual. + +``` +jfrconv --cpu --diff baseline.jfr new.jfr diff.html +``` + +Output file name is optional. If omitted, `jfrconv` takes the name +of the second input file, replacing its extension with `.diff.html`. ## Standalone converter examples diff --git a/src/converter/one/convert/Arguments.java b/src/converter/one/convert/Arguments.java index 5b1a01b6..04895a06 100644 --- a/src/converter/one/convert/Arguments.java +++ b/src/converter/one/convert/Arguments.java @@ -24,6 +24,7 @@ public class Arguments { public boolean help; public boolean reverse; public boolean inverted; + public boolean diff; public boolean cpu; public boolean cpuTime; public boolean wall; diff --git a/src/converter/one/convert/FlameGraph.java b/src/converter/one/convert/FlameGraph.java index 0a6ee43a..066a9271 100644 --- a/src/converter/one/convert/FlameGraph.java +++ b/src/converter/one/convert/FlameGraph.java @@ -20,6 +20,7 @@ public class FlameGraph implements Comparator { private static final String[] FRAME_SUFFIX = {"_[0]", "_[j]", "_[i]", "", "", "_[k]", "_[1]"}; private static final byte HAS_SUFFIX = (byte) 0x80; private static final int FLUSH_THRESHOLD = 15000; + private static final long NEW_FRAME_DIFF = Long.MIN_VALUE; private static final Pattern TID_FRAME_PATTERN = Pattern.compile("\\[(.* )?tid=\\d+]"); private final Arguments args; @@ -29,11 +30,14 @@ public class FlameGraph implements Comparator { private String title = "Flame Graph"; private int[] order; + private int[] cpoolMap; private int depth; private int lastLevel; private long lastX; private long lastTotal; + private long lastDiff; private long mintotal; + private long maxdiff = -1; public FlameGraph(Arguments args) { this.args = args; @@ -90,6 +94,8 @@ public class FlameGraph implements Comparator { while (!br.readLine().isEmpty()) ; for (String line; !(line = br.readLine()).isEmpty(); ) { + if (line.startsWith("d=")) continue; // artifact of a differential flame graph + StringTokenizer st = new StringTokenizer(line.substring(2, line.length() - 1), ","); int nameAndType = Integer.parseInt(st.nextToken()); @@ -109,12 +115,10 @@ public class FlameGraph implements Comparator { int titleIndex = nameAndType >>> 3; byte type = (byte) (nameAndType & 7); - if (st.hasMoreTokens() && (type <= TYPE_INLINED || type >= TYPE_C1_COMPILED)) { - type = TYPE_JIT_COMPILED; - } + byte normalizedType = type <= TYPE_INLINED || type >= TYPE_C1_COMPILED ? TYPE_JIT_COMPILED : type; - Frame f = level > 0 || needRebuild ? new Frame(titleIndex, type) : root; - f.self = f.total = total; + Frame f = level > 0 || needRebuild ? new Frame(titleIndex, normalizedType) : root; + fillFrameCounters(f, type, total); if (st.hasMoreTokens()) f.inlined = Long.parseLong(st.nextToken()); if (st.hasMoreTokens()) f.c1 = Long.parseLong(st.nextToken()); if (st.hasMoreTokens()) f.interpreted = Long.parseLong(st.nextToken()); @@ -177,6 +181,26 @@ public class FlameGraph implements Comparator { depth = Math.max(depth, stack.size); } + public void diff(FlameGraph base) { + // Build a map that translates this cpool keys to the base flamegraph's cpool keys + cpoolMap = Arrays.stream(cpool.keys()).mapToInt(title -> base.cpool.getOrDefault(title, -1)).toArray(); + diff(base.root, root); + } + + private void diff(Frame base, Frame current) { + current.diff = base == null ? NEW_FRAME_DIFF : current.self - base.self; + maxdiff = Math.max(maxdiff, Math.abs(current.diff)); + + for (Frame child : current.values()) { + Frame baseChild = base == null ? null : base.get(translateKey(child.key)); + diff(baseChild, child); + } + } + + private int translateKey(int key) { + return cpoolMap[key & TITLE_MASK] | (key & ~TITLE_MASK); + } + public void dump(OutputStream out) throws IOException { try (PrintStream ps = new PrintStream(out, false, "UTF-8")) { dump(ps); @@ -205,6 +229,9 @@ public class FlameGraph implements Comparator { tail = printTill(out, tail, "/*inverted:*/false"); out.print(args.reverse ^ args.inverted); + tail = printTill(out, tail, "/*maxdiff:*/-1"); + out.print(maxdiff); + tail = printTill(out, tail, "/*depth:*/0"); out.print(depth); @@ -239,6 +266,15 @@ public class FlameGraph implements Comparator { } private void printFrame(PrintStream out, Frame frame, int level, long x) { + StringBuilder sb = outbuf; + if (frame.diff != lastDiff) { + if (frame.diff == NEW_FRAME_DIFF) { + sb.append("d=U\n"); + } else { + sb.append("d=").append(frame.diff).append('\n'); + } + } + int nameAndType = order[frame.getTitleIndex()] << 3 | frame.getType(); boolean hasExtraTypes = (frame.inlined | frame.c1 | frame.interpreted) != 0 && frame.inlined < frame.total && frame.interpreted < frame.total; @@ -250,7 +286,7 @@ public class FlameGraph implements Comparator { func = 'n'; } - StringBuilder sb = outbuf.append(func).append('(').append(nameAndType); + sb.append(func).append('(').append(nameAndType); if (func == 'f') { sb.append(',').append(level).append(',').append(x - lastX); } @@ -270,6 +306,7 @@ public class FlameGraph implements Comparator { lastLevel = level; lastX = x; lastTotal = frame.total; + lastDiff = frame.diff; Frame[] children = frame.values().toArray(EMPTY_FRAME_ARRAY); Arrays.sort(children, this); @@ -291,6 +328,9 @@ public class FlameGraph implements Comparator { sb.append(strings[frame.getTitleIndex()]).append(FRAME_SUFFIX[frame.getType()]); if (frame.self > 0) { int tmpLength = sb.length(); + if (maxdiff >= 0) { + sb.append(' ').append(frame.diff == NEW_FRAME_DIFF ? 0 : frame.self - frame.diff); + } out.print(sb.append(' ').append(frame.self).append('\n')); sb.setLength(tmpLength); } @@ -328,6 +368,21 @@ public class FlameGraph implements Comparator { return include != null; } + private static void fillFrameCounters(Frame frame, byte type, long ticks) { + frame.self = frame.total = ticks; + switch (type) { + case TYPE_INTERPRETED: + frame.interpreted = ticks; + break; + case TYPE_INLINED: + frame.inlined = ticks; + break; + case TYPE_C1_COMPILED: + frame.c1 = ticks; + break; + } + } + private Frame addChild(Frame frame, String title, byte type, long ticks) { frame.total += ticks; diff --git a/src/converter/one/convert/Frame.java b/src/converter/one/convert/Frame.java index 4af9315a..29086d99 100644 --- a/src/converter/one/convert/Frame.java +++ b/src/converter/one/convert/Frame.java @@ -16,11 +16,13 @@ public class Frame extends HashMap { public static final byte TYPE_KERNEL = 5; public static final byte TYPE_C1_COMPILED = 6; - private static final int TYPE_SHIFT = 28; + static final int TYPE_SHIFT = 28; + static final int TITLE_MASK = (1 << TYPE_SHIFT) - 1; final int key; long total; long self; + long diff; long inlined, c1, interpreted; private Frame(int key) { @@ -36,7 +38,7 @@ public class Frame extends HashMap { } int getTitleIndex() { - return key & ((1 << TYPE_SHIFT) - 1); + return key & TITLE_MASK; } byte getType() { diff --git a/src/converter/one/convert/Main.java b/src/converter/one/convert/Main.java index d48ef199..4e5eefd0 100644 --- a/src/converter/one/convert/Main.java +++ b/src/converter/one/convert/Main.java @@ -7,6 +7,7 @@ package one.convert; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; public class Main { @@ -18,7 +19,7 @@ public class Main { return; } - if (args.files.size() == 1) { + if (args.files.size() == (args.diff ? 2 : 1)) { args.files.add("."); } @@ -35,6 +36,34 @@ public class Main { } } + if (args.diff) { + if (fileCount != 2) { + throw new IllegalArgumentException("--diff option requires two input files"); + } + if (!"html".equals(args.output) && !"collapsed".equals(args.output)) { + throw new IllegalArgumentException("--diff option requires html or collapsed output format"); + } + + args.norm = true; // don't let random IDs in class names spoil comparison + + String input1 = args.files.get(0); + String input2 = args.files.get(1); + String output = isDirectory ? new File(lastFile, replaceExt(input2, "diff." + args.output)).getPath() : lastFile; + + System.out.print("Converting " + getFileName(input2) + " vs " + getFileName(input1) + " -> " + getFileName(output) + " "); + System.out.flush(); + + long startTime = System.nanoTime(); + FlameGraph base = parseFlameGraph(input1, args); + FlameGraph current = parseFlameGraph(input2, args); + current.diff(base); + current.dump(new FileOutputStream(output)); + long endTime = System.nanoTime(); + + System.out.print("# " + (endTime - startTime) / 1000000 / 1000.0 + " s\n"); + return; + } + for (int i = 0; i < fileCount; i++) { String input = args.files.get(i); String output = isDirectory ? new File(lastFile, replaceExt(input, args.output)).getPath() : lastFile; @@ -106,6 +135,7 @@ public class Main { " -o --output FORMAT Output format: html, collapsed, pprof, pb.gz, heatmap, otlp\n" + " -I --include REGEX Include only stacks with the specified frames\n" + " -X --exclude REGEX Exclude stacks with the specified frames\n" + + " --diff Create differential Flame Graph from two input files\n" + "\n" + "JFR options:\n" + " --cpu CPU profile (ExecutionSample)\n" + diff --git a/src/res/flame.html b/src/res/flame.html index 108a4225..c3a9b70a 100644 --- a/src/res/flame.html +++ b/src/res/flame.html @@ -75,9 +75,11 @@ // SPDX-License-Identifier: Apache-2.0 'use strict'; let root, px, pattern; - let level0 = 0, left0 = 0, width0 = 0; + let level0 = 0, left0 = 0, width0 = 0, d = 0; let nav = [], navIndex, matchval; let inverted = /*inverted:*/false; + const U = undefined; + const maxdiff = /*maxdiff:*/-1; const levels = Array(/*depth:*/0); for (let h = 0; h < levels.length; h++) { levels[h] = []; @@ -111,10 +113,18 @@ return '#' + (p[0] + ((p[1] * v) << 16 | (p[2] * v) << 8 | (p[3] * v))).toString(16); } + function getDiffColor(diff) { + if (diff === U) return '#ffdd33'; + if (diff === 0) return '#e0e0e0'; + const v = Math.round(128 * (maxdiff - Math.abs(diff)) / maxdiff) + 96; + return diff > 0 ? 'rgb(255,' + v + ',' + v + ')' : 'rgb(' + v + ',' + v + ',255)'; + } + function f(key, level, left, width, inln, c1, int) { levels[level0 = level].push({level, left: left0 += left, width: width0 = width || width0, - color: getColor(palette[key & 7]), title: cpool[key >>> 3], - details: (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '') + color: maxdiff >= 0 ? getDiffColor(d) : getColor(palette[key & 7]), + title: cpool[key >>> 3], + details: (d ? (d > 0 ? ', +' : ', ') + d : '') + (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '') }); } diff --git a/test/test/jfrconverter/JfrconverterTests.java b/test/test/jfrconverter/JfrconverterTests.java index 5cd5bda0..7afa0124 100644 --- a/test/test/jfrconverter/JfrconverterTests.java +++ b/test/test/jfrconverter/JfrconverterTests.java @@ -5,13 +5,19 @@ package test.jfrconverter; -import test.otlp.CpuBurner; import one.convert.*; import one.jfr.JfrReader; +import one.jfr.StackTrace; import one.jfr.event.Event; import one.jfr.event.EventCollector; -import one.jfr.StackTrace; -import one.profiler.test.*; +import one.profiler.test.Output; +import one.profiler.test.Test; +import one.profiler.test.TestProcess; +import test.otlp.CpuBurner; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; // Simple smoke tests for JFR converter. The output is not inspected for errors, // we only verify that the conversion completes successfully. @@ -72,4 +78,33 @@ public class JfrconverterTests { assert !found[3]; } } + + @Test(mainClass = Main.class, args = "--diff test/test/jfrconverter/sample1.collapsed test/test/jfrconverter/sample2.collapsed %diff.collapsed") + public void diffCollapsed(TestProcess p) throws Exception { + Output out = p.waitForExit("%diff"); + assert out.containsExact("BusyClient.run_[j] 4 1"); + assert out.containsExact("BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2 2"); + assert out.containsExact("ByteBuffer.get_[i];ByteBuffer.getArray_[i] 0 1"); + assert out.samples("ByteBuffer.get") == 2; + } + + @Test(mainClass = Main.class, args = "--diff test/test/jfrconverter/sample1.collapsed test/test/jfrconverter/sample2.collapsed %diff.html") + public void diffHtml(TestProcess p) throws Exception { + Output out = p.waitForExit("%diff"); + assert out.containsExact("d=-3"); + assert out.containsExact("d=0"); + assert out.containsExact("d=U"); + + // It should be possible to reconstruct original FlameGraph from the differential one + byte[] original = buildFlameGraph("test/test/jfrconverter/sample2.collapsed"); + byte[] reconstructed = buildFlameGraph(p.getFilePath("%diff")); + assert Arrays.equals(original, reconstructed); + } + + private static byte[] buildFlameGraph(String input) throws IOException { + FlameGraph fg = FlameGraph.parse(input, new Arguments()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + fg.dump(baos); + return baos.toByteArray(); + } } diff --git a/test/test/jfrconverter/sample1.collapsed b/test/test/jfrconverter/sample1.collapsed new file mode 100644 index 00000000..b84c8ffd --- /dev/null +++ b/test/test/jfrconverter/sample1.collapsed @@ -0,0 +1,17 @@ +BusyClient.run_[j] 4 +BusyClient.run_[j];InputStream.read_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i];NativeThread.current_[i];NativeThread.current0_[j] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.endRead_[i] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];CarrierThreadLocal.get_[i];System$2.getCarrierThreadLocal_[i];ThreadLocal.getCarrierThreadLocal_[i];jlong_disjoint_arraycopy 15 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];Util$BufferCache.get_[i];Buffer.capacity_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0;read 143 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i];ReentrantLock$Sync.lock_[i];ReentrantLock$NonfairSync.initialTryLock_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i];ReentrantLock$Sync.lock_[i];ReentrantLock$NonfairSync.initialTryLock_[i];AbstractQueuedSynchronizer.compareAndSetState_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.unlock_[i];AbstractQueuedSynchronizer.release_[i];ReentrantLock$Sync.tryRelease_[i];AbstractQueuedSynchronizer.setState_[i] 1 diff --git a/test/test/jfrconverter/sample2.collapsed b/test/test/jfrconverter/sample2.collapsed new file mode 100644 index 00000000..20ccb7a4 --- /dev/null +++ b/test/test/jfrconverter/sample2.collapsed @@ -0,0 +1,17 @@ +BusyClient.run_[j] 1 +BusyClient.run_[j];InputStream.read_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i] 3 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i];NativeThread.current_[i];NativeThread.current0_[j] 4 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.endRead_[i] 4 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];ByteBuffer.get_[i];ByteBuffer.getArray_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];ByteBuffer.get_[i];Buffer.position_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];CarrierThreadLocal.get_[i];System$2.getCarrierThreadLocal_[i];ThreadLocal.getCarrierThreadLocal_[i];jlong_disjoint_arraycopy 6 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j] 4 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0;read 151 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i] 1 +BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.unlock_[i];AbstractQueuedSynchronizer.release_[i];ReentrantLock$Sync.tryRelease_[i];AbstractQueuedSynchronizer.setState_[i] 3