Remove unused GPU usage tools
This commit is contained in:
Reuben Morais
parent
a05baa35c9
commit
20b0ab17ea
@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
from deepspeech_training.util.gpu_usage import GPUUsage
|
||||
|
||||
gu = GPUUsage()
|
||||
gu.start()
|
||||
@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
import sys
|
||||
|
||||
from deepspeech_training.util.gpu_usage import GPUUsageChart
|
||||
|
||||
GPUUsageChart(sys.argv[1], sys.argv[2])
|
||||
@ -1,159 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import csv
|
||||
|
||||
from threading import Thread
|
||||
from time import time
|
||||
from scipy.interpolate import spline
|
||||
|
||||
from six.moves import range
|
||||
# Do this to be able to use without X
|
||||
import matplotlib as mpl
|
||||
mpl.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
class GPUUsage(Thread):
|
||||
def __init__(self, csvfile=None):
|
||||
super(GPUUsage, self).__init__()
|
||||
|
||||
self._cmd = [ 'nvidia-smi', 'dmon', '-d', '1', '-s', 'pucvmet' ]
|
||||
self._names = []
|
||||
self._units = []
|
||||
self._process = None
|
||||
|
||||
self._csv_output = csvfile or os.environ.get('ds_gpu_usage_csv', self.make_basename(prefix='ds-gpu-usage', extension='csv'))
|
||||
|
||||
def get_git_desc(self):
|
||||
return subprocess.check_output(['git', 'describe', '--always', '--abbrev']).strip()
|
||||
|
||||
def make_basename(self, prefix, extension):
|
||||
# Let us assume that this code is executed in the current git clone
|
||||
return '%s.%s.%s.%s' % (prefix, self.get_git_desc(), int(time()), extension)
|
||||
|
||||
def stop(self):
|
||||
if not self._process:
|
||||
print("Trying to stop nvidia-smi but no more process, please fix.")
|
||||
return
|
||||
|
||||
print("Ending nvidia-smi monitoring: PID", self._process.pid)
|
||||
self._process.terminate()
|
||||
print("Ended nvidia-smi monitoring ...")
|
||||
|
||||
def run(self):
|
||||
print("Starting nvidia-smi monitoring")
|
||||
|
||||
# If the system has no CUDA setup, then this will fail.
|
||||
try:
|
||||
self._process = subprocess.Popen(self._cmd, stdout=subprocess.PIPE)
|
||||
except OSError as ex:
|
||||
print("Unable to start monitoring, check your environment:", ex)
|
||||
return
|
||||
|
||||
writer = None
|
||||
with open(self._csv_output, 'w') as f:
|
||||
for line in iter(self._process.stdout.readline, ''):
|
||||
d = self.ingest(line)
|
||||
|
||||
if line.startswith('# '):
|
||||
if len(self._names) == 0:
|
||||
self._names = d
|
||||
writer = csv.DictWriter(f, delimiter=str(','), quotechar=str('"'), fieldnames=d)
|
||||
writer.writeheader()
|
||||
continue
|
||||
if len(self._units) == 0:
|
||||
self._units = d
|
||||
continue
|
||||
else:
|
||||
assert len(self._names) == len(self._units)
|
||||
assert len(d) == len(self._names)
|
||||
assert len(d) > 1
|
||||
writer.writerow(self.merge_line(d))
|
||||
f.flush()
|
||||
|
||||
def ingest(self, line):
|
||||
return map(lambda x: x.replace('-', '0'), filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(' ')[1:])))
|
||||
|
||||
def merge_line(self, line):
|
||||
return dict(zip(self._names, line))
|
||||
|
||||
class GPUUsageChart():
|
||||
def __init__(self, source, basename=None):
|
||||
self._rows = [ 'pwr', 'temp', 'sm', 'mem']
|
||||
self._titles = {
|
||||
'pwr': "Power (W)",
|
||||
'temp': "Temperature (°C)",
|
||||
'sm': "Streaming Multiprocessors (%)",
|
||||
'mem': "Memory (%)"
|
||||
}
|
||||
self._data = { }.fromkeys(self._rows)
|
||||
self._csv = source
|
||||
self._basename = basename or os.environ.get('ds_gpu_usage_charts', 'gpu_usage_%%s_%d.png' % int(time.time()))
|
||||
|
||||
# This should make sure we start from anything clean.
|
||||
plt.close("all")
|
||||
|
||||
try:
|
||||
self.read()
|
||||
for plot in self._rows:
|
||||
self.produce_plot(plot)
|
||||
except IOError as ex:
|
||||
print("Unable to read", ex)
|
||||
|
||||
def append_data(self, row):
|
||||
for bucket, value in row.iteritems():
|
||||
if not bucket in self._rows:
|
||||
continue
|
||||
|
||||
if not self._data[bucket]:
|
||||
self._data[bucket] = {}
|
||||
|
||||
gpu = int(row['gpu'])
|
||||
if not self._data[bucket].has_key(gpu):
|
||||
self._data[bucket][gpu] = [ value ]
|
||||
else:
|
||||
self._data[bucket][gpu] += [ value ]
|
||||
|
||||
def read(self):
|
||||
print("Reading data from", self._csv)
|
||||
with open(self._csv, 'r') as f:
|
||||
for r in csv.DictReader(f):
|
||||
self.append_data(r)
|
||||
|
||||
def produce_plot(self, key, with_spline=True):
|
||||
png = self._basename % (key, )
|
||||
print("Producing plot for", key, "as", png)
|
||||
fig, axis = plt.subplots()
|
||||
data = self._data[key]
|
||||
if data is None:
|
||||
print("Data was empty, aborting")
|
||||
return
|
||||
|
||||
x = list(range(len(data[0])))
|
||||
if with_spline:
|
||||
x = map(lambda x: float(x), x)
|
||||
x_sm = np.array(x)
|
||||
x_smooth = np.linspace(x_sm.min(), x_sm.max(), 300)
|
||||
|
||||
for gpu, y in data.iteritems():
|
||||
if with_spline:
|
||||
y = map(lambda x: float(x), y)
|
||||
y_sm = np.array(y)
|
||||
y_smooth = spline(x, y, x_smooth, order=1)
|
||||
axis.plot(x_smooth, y_smooth, label='GPU %d' % (gpu))
|
||||
else:
|
||||
axis.plot(x, y, label='GPU %d' % (gpu))
|
||||
|
||||
axis.legend(loc="upper right", frameon=False)
|
||||
axis.set_xlabel("Time (s)")
|
||||
axis.set_ylabel("%s" % self._titles[key])
|
||||
fig.set_size_inches(24, 18)
|
||||
plt.title("GPU Usage: %s" % self._titles[key])
|
||||
plt.savefig(png, dpi=100)
|
||||
plt.close(fig)
|
||||
Loading…
Reference in New Issue
Block a user