Get asynchronous thread dump (jstack)

profiler.sh

@@ -7,6 +7,7 @@ usage() {
     echo "  start             start profiling and return immediately"
     echo "  resume            resume profiling without resetting collected data"
     echo "  stop              stop profiling"
+    echo "  jstack            get a thread dump"
     echo "  check             check if the specified profiling event is available"
     echo "  status            print profiling status"
     echo "  list              list profiling events supported by the target JVM"
@@ -109,6 +110,11 @@ while [ $# -gt 0 ]; do
         start|resume|stop|check|status|list|collect)
             ACTION="$1"
             ;;
+        jstack)
+            ACTION="start"
+            EVENT="jstack"
+            PARAMS="$PARAMS,threads"
+            ;;
         -v|--version)
             ACTION="version"
             ;;

src/arguments.h

@@ -28,6 +28,7 @@ const char* const EVENT_ALLOC  = "alloc";
 const char* const EVENT_LOCK   = "lock";
 const char* const EVENT_WALL   = "wall";
 const char* const EVENT_ITIMER = "itimer";
+const char* const EVENT_JSTACK = "jstack";
 
 enum Action {
     ACTION_NONE,

src/jstack.cpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2021 Andrei Pangin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <signal.h>
+#include <time.h>
+#include "jstack.h"
+#include "profiler.h"
+
+
+// Wait at most this number of milliseconds to finish processing of pending signals
+const int MAX_WAIT_MILLIS = 2000;
+
+
+void JStack::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) {
+    ExecutionEvent event;
+    event._thread_state = Profiler::_instance.getThreadState(ucontext);
+    Profiler::_instance.recordSample(ucontext, 1, 0, &event);
+}
+
+Error JStack::start(Arguments& args) {
+    OS::installSignalHandler(SIGVTALRM, signalHandler);
+
+    int self = OS::threadId();
+    u64 required_samples = Profiler::_instance.total_samples();
+    ThreadFilter* thread_filter = Profiler::_instance.threadFilter();
+    bool thread_filter_enabled = thread_filter->enabled();
+
+    ThreadList* thread_list = OS::listThreads();
+    int thread_id;
+    while ((thread_id = thread_list->next()) != -1) {
+        if (thread_id != self && (!thread_filter_enabled || thread_filter->accept(thread_id))) {
+            if (OS::sendSignalToThread(thread_id, SIGVTALRM)) {
+                required_samples++;
+            }
+        }
+    }
+    delete thread_list;
+
+    // Get our own stack trace after all other threads
+    if (!thread_filter_enabled || thread_filter->accept(self)) {
+        ExecutionEvent event;
+        event._thread_state = THREAD_RUNNING;
+        Profiler::_instance.recordSample(NULL, 1, 0, &event);
+        required_samples++;
+    }
+
+    // Wait until all asynchronous stack traces collected
+    for (int i = 0; Profiler::_instance.total_samples() < required_samples && i < MAX_WAIT_MILLIS; i++) {
+        struct timespec timeout = {0, 1000000};
+        nanosleep(&timeout, NULL);
+    }
+
+    return Error::OK;
+}
+
+void JStack::stop() {
+    // Nothing to do
+}

src/jstack.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2021 Andrei Pangin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _JSTACK_H
+#define _JSTACK_H
+
+#include <signal.h>
+#include "engine.h"
+
+
+class JStack : public Engine {
+  private:
+    static void signalHandler(int signo, siginfo_t* siginfo, void* ucontext);
+
+  public:
+    const char* name() {
+        return EVENT_JSTACK;
+    }
+
+    const char* units() {
+        return "samples";
+    }
+
+    Error start(Arguments& args);
+    void stop();
+};
+
+#endif // _JSTACK_H

src/profiler.cpp

@@ -18,6 +18,7 @@
 #include <fstream>
 #include <dlfcn.h>
 #include <unistd.h>
+#include <sched.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -30,6 +31,7 @@
 #include "wallClock.h"
 #include "instrument.h"
 #include "itimer.h"
+#include "jstack.h"
 #include "flameGraph.h"
 #include "flightRecorder.h"
 #include "frameName.h"
@@ -47,6 +49,7 @@ static AllocTracer alloc_tracer;
 static LockTracer lock_tracer;
 static WallClock wall_clock;
 static ITimer itimer;
+static JStack jstack;
 static Instrument instrument;
 
 
@@ -491,15 +494,42 @@ AddressType Profiler::getAddressType(instruction_t* pc) {
     return ADDR_UNKNOWN;
 }
 
+ThreadState Profiler::getThreadState(void* ucontext) {
+    StackFrame frame(ucontext);
+    uintptr_t pc = frame.pc();
+
+    // Consider a thread sleeping, if it has been interrupted in the middle of syscall execution,
+    // either when PC points to the syscall instruction, or if syscall has just returned with EINTR
+    if (StackFrame::isSyscall((instruction_t*)pc)) {
+        return THREAD_SLEEPING;
+    }
+
+    // Make sure the previous instruction address is readable
+    uintptr_t prev_pc = pc - SYSCALL_SIZE;
+    if ((pc & 0xfff) >= SYSCALL_SIZE || findNativeLibrary((instruction_t*)prev_pc) != NULL) {
+        if (StackFrame::isSyscall((instruction_t*)prev_pc) && frame.checkInterruptedSyscall()) {
+            return THREAD_SLEEPING;
+        }
+    }
+
+    return THREAD_RUNNING;
+}
+
 void Profiler::recordSample(void* ucontext, u64 counter, jint event_type, Event* event) {
     atomicInc(_total_samples);
 
     int tid = OS::threadId();
     u32 lock_index = getLockIndex(tid);
-    if (!_locks[lock_index].tryLock() &&
-        !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
-        !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())
+    while (!_locks[lock_index].tryLock() &&
+           !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() &&
+           !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock())
     {
+        if (_engine == &jstack) {
+            // JStack strictly needs all stack traces, but we cannot lock inside a signal handler
+            sched_yield();
+            continue;
+        }
+
         // Too many concurrent signals already
         atomicInc(_failures[-ticks_skipped]);
 
@@ -798,6 +828,8 @@ Engine* Profiler::selectEngine(const char* event_name) {
         return &wall_clock;
     } else if (strcmp(event_name, EVENT_ITIMER) == 0) {
         return &itimer;
+    } else if (strcmp(event_name, EVENT_JSTACK) == 0) {
+        return &jstack;
     } else if (strchr(event_name, '.') != NULL) {
         return &instrument;
     } else {
@@ -908,9 +940,11 @@ Error Profiler::start(Arguments& args, bool reset) {
     if (_events & EK_ALLOC) alloc_tracer.start(args);
     if (_events & EK_LOCK) lock_tracer.start(args);
 
-    // Thread events might be already enabled by PerfEvents::start
-    switchThreadEvents(JVMTI_ENABLE);
-    switchNativeMethodTraps(true);
+    if (_engine != &jstack) {
+        // Thread events might be already enabled by PerfEvents::start
+        switchThreadEvents(JVMTI_ENABLE);
+        switchNativeMethodTraps(true);
+    }
 
     _state = RUNNING;
     _start_time = time(NULL);
@@ -930,8 +964,11 @@ Error Profiler::stop() {
 
     _engine->stop();
 
-    switchNativeMethodTraps(false);
-    switchThreadEvents(JVMTI_DISABLE);
+    if (_engine != &jstack) {
+        switchNativeMethodTraps(false);
+        switchThreadEvents(JVMTI_DISABLE);
+    }
+
     updateJavaThreadNames();
     updateNativeThreadNames();
 
@@ -968,6 +1005,25 @@ void Profiler::switchThreadEvents(jvmtiEventMode mode) {
     }
 }
 
+void Profiler::dump(std::ostream& out, Arguments& args) {
+    switch (args._output) {
+        case OUTPUT_COLLAPSED:
+            dumpCollapsed(out, args);
+            break;
+        case OUTPUT_FLAMEGRAPH:
+            dumpFlameGraph(out, args, false);
+            break;
+        case OUTPUT_TREE:
+            dumpFlameGraph(out, args, true);
+            break;
+        case OUTPUT_FLAT:
+            dumpFlat(out, args);
+            break;
+        default:
+            break;
+    }
+}
+
 /*
  * Dump stacks in FlameGraph input format:
  * 
@@ -1097,6 +1153,9 @@ void Profiler::runInternal(Arguments& args, std::ostream& out) {
             Error error = start(args, args._action == ACTION_START);
             if (error) {
                 out << error.message() << std::endl;
+            } else if (_engine == &jstack) {
+                stop();
+                dump(out, args);
             } else {
                 out << "Profiling started" << std::endl;
             }
@@ -1160,22 +1219,7 @@ void Profiler::runInternal(Arguments& args, std::ostream& out) {
             break;
         case ACTION_DUMP:
             stop();
-            switch (args._output) {
-                case OUTPUT_COLLAPSED:
-                    dumpCollapsed(out, args);
-                    break;
-                case OUTPUT_FLAMEGRAPH:
-                    dumpFlameGraph(out, args, false);
-                    break;
-                case OUTPUT_TREE:
-                    dumpFlameGraph(out, args, true);
-                    break;
-                case OUTPUT_FLAT:
-                    dumpFlat(out, args);
-                    break;
-                default:
-                    break;
-            }
+            dump(out, args);
             break;
         default:
             break;

src/profiler.h

@@ -188,9 +188,11 @@ class Profiler {
     Error start(Arguments& args, bool reset);
     Error stop();
     void switchThreadEvents(jvmtiEventMode mode);
+    void dump(std::ostream& out, Arguments& args);
     void dumpCollapsed(std::ostream& out, Arguments& args);
     void dumpFlameGraph(std::ostream& out, Arguments& args, bool tree);
     void dumpFlat(std::ostream& out, Arguments& args);
+    ThreadState getThreadState(void* ucontext);
     void recordSample(void* ucontext, u64 counter, jint event_type, Event* event);
 
     void updateSymbols(bool kernel_symbols);

src/wallClock.cpp

@@ -21,7 +21,6 @@
 #include <sys/types.h>
 #include "wallClock.h"
 #include "profiler.h"
-#include "stackFrame.h"
 
 
 // Maximum number of threads sampled in one iteration. This limit serves as a throttle
@@ -41,30 +40,9 @@ const int WAKEUP_SIGNAL = SIGIO;
 long WallClock::_interval;
 bool WallClock::_sample_idle_threads;
 
-ThreadState WallClock::getThreadState(void* ucontext) {
-    StackFrame frame(ucontext);
-    uintptr_t pc = frame.pc();
-
-    // Consider a thread sleeping, if it has been interrupted in the middle of syscall execution,
-    // either when PC points to the syscall instruction, or if syscall has just returned with EINTR
-    if (StackFrame::isSyscall((instruction_t*)pc)) {
-        return THREAD_SLEEPING;
-    }
-
-    // Make sure the previous instruction address is readable
-    uintptr_t prev_pc = pc - SYSCALL_SIZE;
-    if ((pc & 0xfff) >= SYSCALL_SIZE || Profiler::_instance.findNativeLibrary((instruction_t*)prev_pc) != NULL) {
-        if (StackFrame::isSyscall((instruction_t*)prev_pc) && frame.checkInterruptedSyscall()) {
-            return THREAD_SLEEPING;
-        }
-    }
-
-    return THREAD_RUNNING;
-}
-
 void WallClock::signalHandler(int signo, siginfo_t* siginfo, void* ucontext) {
     ExecutionEvent event;
-    event._thread_state = _sample_idle_threads ? getThreadState(ucontext) : THREAD_RUNNING;
+    event._thread_state = _sample_idle_threads ? Profiler::_instance.getThreadState(ucontext) : THREAD_RUNNING;
     Profiler::_instance.recordSample(ucontext, _interval, 0, &event);
 }
 

src/wallClock.h

@@ -21,7 +21,6 @@
 #include <signal.h>
 #include <pthread.h>
 #include "engine.h"
-#include "os.h"
 
 
 class WallClock : public Engine {
@@ -39,7 +38,6 @@ class WallClock : public Engine {
         return NULL;
     }
 
-    static ThreadState getThreadState(void* ucontext);
     static void signalHandler(int signo, siginfo_t* siginfo, void* ucontext);
     static void wakeupHandler(int signo);