Support are probably already looking at your thread dumps already, but I wrote a couple of scripts to help try to diagnose the issues or at least get a better perspective on them.
The first script has the function saveThreadDump() which will use the system.util.threadDump() function to take a thread dump. This is far more useful than dumping from the gateway webpage as it actually has CPU usage against each thread. I attached this to the CPU utilisation system tag change event if it gets above 0.50 with a 30s cooldown - i send myself an email when it runs as well. This in particular has been extremely useful for support as it’s captured logs when the CPU has been high in different scenarios
The second will take a thread dump created by this and convert it into a csv that you can paste into Excel where you can then sort by CPU usage and other things. Note: it does also work with the webpage dump as well. Both dumps are slightly different, but the headers I’ve kept the same so there might be blank values for some headings depending which dump you’ve converted.
I run the second script in PyCharm using Python 3.9. Haven’t checked if it’ll run in Ignition. You’ll need to install the pywin32 library for Windows to get access to the clipboard, otherwise you could always just write the csv contents to a file.
You’ll end up with something like this:
def saveThreadDump():
'''
Description:
Takes a thread dump and records the current CPU, memory, and disk utilisation at the top of the file.
This method is far more useful that using the webpage thread download button as it contains CPU usage per thread.
File is stored local to the client that issues the function call.
'''
now_formatted = system.date.format(system.date.now(), 'yyyyMMdd_HHmmss')
filename = 'IgnitionThreadDump_AU6_{}.txt'.format(now_formatted)
fileFolderPath = 'C:/Ignition Reports/Thread Dumps/'
filePath = '{}{}'.format(fileFolderPath, filename)
dump = system.util.threadDump()
# collect some basic overall performance stats and record them at the top of the file
stats = system.tag.readBlocking(['[System]Gateway/Performance/CPU Usage', '[System]Gateway/Performance/Disk Utilization', '[System]Gateway/Performance/Memory Utilization'])
cpu_util = stats[0].value*100
disk_util = stats[1].value*100
mem_util = stats[2].value*100
dump = 'CPU Utilisation: {:.2f}%\r\nMemory Utilisation: {:.2f}%\r\nDisk Utilisation: {:.2f}%\r\n{}'.format(cpu_util, mem_util, disk_util, dump)
system.file.writeFile(filePath, dump)
return filePath
Tag change script on [System]Gateway/Performance/CPU Usage:
cpu_log_condition = 0.5
if currentValue.value > cpu_log_condition:
now = system.date.now()
lastDump = system.tag.readBlocking(['[default]System/Diagnostic/Last Thread Dump Time'])[0].value
if lastDump is None:
lastDump = system.date.parse('1987-01-01 00:00:00')
dumpLogRate = system.tag.readBlocking(['[default]System/Diagnostic/Thread Dump Log Rate'])[0].value
if system.date.secondsBetween(lastDump, now) > dumpLogRate:
filepath = shared.dev.diag.saveThreadDump()
shared.errors.sendEmail('CPU Usage High {:.1f}%'.format(currentValue.value*100), \
'Thread dump saved to disk with filepath: {}'.format(filepath))
Thread dump converter function to convert to CSV
import sys
import re
import tkinter as tk
from tkinter import filedialog
from tkinter import messagebox
import win32clipboard # part of library: pywin32
def setClipboard(text):
# set clipboard data
win32clipboard.OpenClipboard()
win32clipboard.EmptyClipboard()
win32clipboard.SetClipboardText(text)
win32clipboard.CloseClipboard()
def getClipboard():
# get clipboard data
win32clipboard.OpenClipboard()
data = win32clipboard.GetClipboardData()
win32clipboard.CloseClipboard()
return data
def choose_file():
root = tk.Tk()
root.withdraw()
file_path = filedialog.askopenfilename()
return file_path
def parseThreadDump():
file_path = choose_file()
f = open(file_path, "r")
thread_dump = f.read()
f.close()
thread_dump = thread_dump.splitlines()
thread_id = ''
thread_type = ''
thread_cat = ''
thread_name = ''
thread_cpu = ''
thread_status = ''
thread_jobs = []
headers = ['ID', 'Type', 'Thread Category', 'Thread Name', 'CPU', 'Status', 'Job Count', 'Jobs']
rows = []
# find the ignition version row. Should be at the top, but user may have added additional stuff there such as cpu/mem/disk utilisation
for line_num, line in enumerate(thread_dump):
find = re.findall('(Ignition[a-zA-Z :]+)(\d+.\d+.\d+)( \(b[\d]+\))', line)
if len(find) > 0:
ignition_version_line_num = line_num
ignition_version = '{} {}'.format(find[0][1], find[0][2])
break
# check what type of dump was created (from webpage or system.util.threadDump, as they are formatted slightly differently)
# threads start on the 2nd line after the ignition version row
if 'id=' in thread_dump[ignition_version_line_num+2]:
dump_type = 'webpage'
thread_jobs_prefix = ' '
else:
dump_type = 'script'
thread_jobs_prefix = ' '
for line_num in range(ignition_version_line_num+1, len(thread_dump)):
line = thread_dump[line_num]
# if the line contains thread jobs
if line[0:len(thread_jobs_prefix)] == thread_jobs_prefix:
thread_jobs.append(line.replace(thread_jobs_prefix, ''))
# if the line contains CPU usage (only applicable for dump_type == 'script', webpage doesn't have this info)
elif "CPU: " in line:
thread_cpu = re.findall('[\d.]+', line)[0]
# if the line contains thread status (only applicable for dump_type == 'script', webpage has this in thread line)
elif "java.lang.Thread.State" in line:
thread_status = re.findall('(: )([A-Z_]+)', line)[0][1]
# ignore any blank lines
elif line in ['','"']:
pass
# else assume it contains a thread definition
else:
# if a previous thread is currently in memory, push it into the rows list
if thread_status != '':
thread_details = [thread_id,
thread_type,
thread_cat,
thread_name,
thread_cpu,
thread_status,
str(len(thread_jobs)),
thread_jobs]
rows.append(thread_details)
# clear the variables
thread_id = ''
thread_type = ''
thread_cat = ''
thread_name = ''
thread_cpu = ''
thread_status = ''
thread_jobs = []
# record the current thread's info
try:
if dump_type == 'webpage':
thread_id = re.findall('id=(\d+),', line)[0]
thread_type = re.findall('^([a-zA-Z ]+) \[', line)[0]
thread_name = re.findall('\[([\w -./:\d@]+)\]', line)[0]
thread_status = re.findall('\(([\w_]+)\)', line)[0]
thread_cat = re.findall('([\w -./:\d@]+)-[\d]+', thread_name)
thread_cat = thread_cat[0] if len(thread_cat) > 0 else thread_name
elif dump_type == 'script':
thread_name = line.replace('"', '')
thread_id = re.findall('-([\d]+)"', line)
if len(thread_id) == 0:
thread_id = ''
else:
thread_id = thread_id[0]
thread_cat = thread_name.replace("-{}".format(thread_id), '')
except Exception as e:
print(e)
print('thread_name=' + thread_name)
sys.exit()
if thread_id == '16':
pass
# if a previous thread is currently in memory, push it into the rows list
if thread_status != '':
thread_details = [thread_id,
thread_type,
thread_cat,
thread_name,
thread_cpu,
thread_status,
str(len(thread_jobs)),
thread_jobs]
rows.append(thread_details)
table = [headers]
table.extend(rows)
return table
def convertThreadArrayToExcelTable(rows):
headers = rows[0]
text = '"' + '","'.join(headers) + '"\r\n'
data = rows[1:]
for row in data:
# replace the jobs list with a text version with new lines for each job
row[-1] = '\r\n'.join(row[-1])
for row in data:
try:
text += '"' + '","'.join(row) + '"\r\n'
except Exception as e:
print(e)
return text
table = convertThreadArrayToExcelTable(parseThreadDump())
setClipboard(table)
messagebox.showinfo(title='Extracted Thread Dump', message='Extracted thread dump into a table. Copied to clipboard to paste into Excel.')