import os, shutil, tempfile, logging
import common
from autotest_lib.client.common_lib import utils, error, profiler_manager
from autotest_lib.server import profiler, autotest, standalone_profiler, hosts
PROFILER_TMPDIR = '/tmp/profilers'
def get_profiler_results_dir(autodir):
"""
Given the directory of the autotest client used to run a profiler,
return the remote path where profiler results will be stored.
"""
return os.path.join(autodir, 'results', 'default', 'profiler_sync',
'profiling')
def get_profiler_log_path(autodir):
"""
Given the directory of a profiler client, find the client log path.
"""
return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG')
class profilers(profiler_manager.profiler_manager):
def __init__(self, job):
super(profilers, self).__init__(job)
self.add_log = {}
self.start_delay = 0
# maps hostname to (host object, autotest.Autotest object, Autotest
# install dir), where the host object is the one created specifically
# for profiling
self.installed_hosts = {}
self.current_test = None
def set_start_delay(self, start_delay):
self.start_delay = start_delay
def load_profiler(self, profiler_name, args, dargs):
newprofiler = profiler.profiler_proxy(profiler_name)
newprofiler.initialize(*args, **dargs)
newprofiler.setup(*args, **dargs) # lazy setup is done client-side
return newprofiler
def add(self, profiler, *args, **dargs):
super(profilers, self).add(profiler, *args, **dargs)
self.add_log[profiler] = (args, dargs)
def delete(self, profiler):
super(profilers, self).delete(profiler)
if profiler in self.add_log:
del self.add_log[profiler]
def _install_clients(self):
"""
Install autotest on any current job hosts.
"""
in_use_hosts = set()
# find hosts in use but not used by us
for host in self.job.hosts:
if host.hostname not in self.job.machines:
# job.hosts include all host instances created on the fly.
# We only care DUTs in job.machines which are
# piped in from autoserv -m option.
continue
autodir = host.get_autodir()
if not (autodir and autodir.startswith(PROFILER_TMPDIR)):
in_use_hosts.add(host.hostname)
logging.debug('Hosts currently in use: %s', in_use_hosts)
# determine what valid host objects we already have installed
profiler_hosts = set()
for host, at, profiler_dir in self.installed_hosts.values():
if host.path_exists(profiler_dir):
profiler_hosts.add(host.hostname)
else:
# the profiler was wiped out somehow, drop this install
logging.warning('The profiler client on %s at %s was deleted',
host.hostname, profiler_dir)
host.close()
del self.installed_hosts[host.hostname]
logging.debug('Hosts with profiler clients already installed: %s',
profiler_hosts)
# install autotest on any new hosts in use
for hostname in in_use_hosts - profiler_hosts:
host = hosts.create_host(hostname)
tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR)
at = autotest.Autotest(host)
at.install_no_autoserv(autodir=tmp_dir)
self.installed_hosts[host.hostname] = (host, at, tmp_dir)
# drop any installs from hosts no longer in job.hosts
hostnames_to_drop = profiler_hosts - in_use_hosts
hosts_to_drop = [self.installed_hosts[hostname][0]
for hostname in hostnames_to_drop]
for host in hosts_to_drop:
host.close()
del self.installed_hosts[host.hostname]
def _get_hosts(self, host=None):
"""
Returns a list of (Host, Autotest, install directory) tuples for hosts
currently supported by this profiler. The returned Host object is always
the one created by this profiler, regardless of what's passed in. If
'host' is not None, all entries not matching that host object are
filtered out of the list.
"""
if host is None:
return self.installed_hosts.values()
if host.hostname in self.installed_hosts:
return [self.installed_hosts[host.hostname]]
return []
def _get_local_profilers_dir(self, test, hostname):
in_machine_dir = (
os.path.basename(test.job.resultdir) in test.job.machines)
if len(test.job.machines) > 1 and not in_machine_dir:
local_dir = os.path.join(test.profdir, hostname)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
else:
local_dir = test.profdir
return local_dir
def _get_failure_logs(self, autodir, test, host):
"""
Collect the client logs from a profiler run and put them in a
file named failure-*.log.
"""
try:
fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-',
dir=self._get_local_profilers_dir(test, host.hostname))
os.close(fd)
host.get_file(get_profiler_log_path(autodir), path)
# try to collect any partial profiler logs
self._get_profiler_logs(autodir, test, host)
except (error.AutotestError, error.AutoservError):
logging.exception('Profiler failure log collection failed')
# swallow the exception so that we don't override an existing
# exception being thrown
def _get_all_failure_logs(self, test, hosts):
for host, at, autodir in hosts:
self._get_failure_logs(autodir, test, host)
def _get_profiler_logs(self, autodir, test, host):
results_dir = get_profiler_results_dir(autodir)
local_dir = self._get_local_profilers_dir(test, host.hostname)
self.job.remove_client_log(host.hostname, results_dir, local_dir)
tempdir = tempfile.mkdtemp(dir=self.job.tmpdir)
try:
host.get_file(results_dir + '/', tempdir)
except error.AutoservRunError:
pass # no files to pull back, nothing we can do
utils.merge_trees(tempdir, local_dir)
shutil.rmtree(tempdir, ignore_errors=True)
def _run_clients(self, test, hosts):
"""
We initialize the profilers just before start because only then we
know all the hosts involved.
"""
hostnames = [host_info[0].hostname for host_info in hosts]
profilers_args = [(p.name, p.args, p.dargs)
for p in self.list]
for host, at, autodir in hosts:
control_script = standalone_profiler.generate_test(hostnames,
host.hostname,
profilers_args,
180, None)
try:
at.run(control_script, background=True)
except Exception:
self._get_failure_logs(autodir, test, host)
raise
remote_results_dir = get_profiler_results_dir(autodir)
local_results_dir = self._get_local_profilers_dir(test,
host.hostname)
self.job.add_client_log(host.hostname, remote_results_dir,
local_results_dir)
try:
# wait for the profilers to be added
standalone_profiler.wait_for_profilers(hostnames)
except Exception:
self._get_all_failure_logs(test, hosts)
raise
def before_start(self, test, host=None):
# create host objects and install the needed clients
# so later in start() we don't spend too much time
self._install_clients()
self._run_clients(test, self._get_hosts(host))
def start(self, test, host=None):
hosts = self._get_hosts(host)
# wait for the profilers to start
hostnames = [host_info[0].hostname for host_info in hosts]
try:
standalone_profiler.start_profilers(hostnames)
except Exception:
self._get_all_failure_logs(test, hosts)
raise
self.current_test = test
def stop(self, test):
assert self.current_test == test
hosts = self._get_hosts()
# wait for the profilers to stop
hostnames = [host_info[0].hostname for host_info in hosts]
try:
standalone_profiler.stop_profilers(hostnames)
except Exception:
self._get_all_failure_logs(test, hosts)
raise
def report(self, test, host=None):
assert self.current_test == test
hosts = self._get_hosts(host)
# when running on specific hosts we cannot wait for the other
# hosts to sync with us
if not host:
hostnames = [host_info[0].hostname for host_info in hosts]
try:
standalone_profiler.finish_profilers(hostnames)
except Exception:
self._get_all_failure_logs(test, hosts)
raise
# pull back all the results
for host, at, autodir in hosts:
self._get_profiler_logs(autodir, test, host)
def handle_reboot(self, host):
if self.current_test:
test = self.current_test
for profiler in self.list:
if not profiler.supports_reboot:
msg = 'profiler %s does not support rebooting during tests'
msg %= profiler.name
self.job.record('WARN', os.path.basename(test.outputdir),
None, msg)
self.report(test, host)
self.before_start(test, host)
self.start(test, host)