diff --git a/.gitignore b/.gitignore index 1296a7fe..e41d4738 100755 --- a/.gitignore +++ b/.gitignore @@ -14,17 +14,10 @@ wa_output/ doc/source/api/ doc/source/plugins/ MANIFEST -wlauto/external/uiautomator/bin/ -wlauto/external/uiautomator/*.properties -wlauto/external/uiautomator/build.xml *.orig local.properties -wlauto/external/revent/libs/ -wlauto/external/revent/obj/ -wlauto/external/bbench_server/libs/ -wlauto/external/bbench_server/obj/ pmu_logger.mod.c .tmp_versions obj/ libs/armeabi -wlauto/workloads/*/uiauto/bin/ +uiauto/bin/ diff --git a/wa/__init__.py b/wa/__init__.py index cf6a4d2e..d005ef27 100644 --- a/wa/__init__.py +++ b/wa/__init__.py @@ -14,4 +14,4 @@ from wa.framework.plugin import Plugin, Parameter from wa.framework.processor import ResultProcessor from wa.framework.resource import (NO_ONE, JarFile, ApkFile, ReventFile, File, Executable) -from wa.framework.workload import Workload +from wa.framework.workload import Workload, ApkUiautoWorkload diff --git a/wa/framework/execution.py b/wa/framework/execution.py index 12b7be8a..d9fc5132 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -30,7 +30,7 @@ import wa.framework.signal as signal from wa.framework import instrumentation, pluginloader from wa.framework.configuration.core import settings, Status from wa.framework.exception import (WAError, ConfigError, TimeoutError, - InstrumentError, TargetError, + InstrumentError, TargetError, HostError, TargetNotRespondingError) from wa.framework.output import init_job_output from wa.framework.plugin import Artifact @@ -138,16 +138,21 @@ class ExecutionContext(object): self.current_job = self.job_queue.pop(0) self.current_job.output = init_job_output(self.run_output, self.current_job) self.update_job_state(self.current_job) + self.tm.start() return self.current_job def end_job(self): if not self.current_job: raise RuntimeError('No jobs in progress') + self.tm.stop() self.completed_jobs.append(self.current_job) self.update_job_state(self.current_job) self.output.write_result() self.current_job = None + def extract_results(self): + self.tm.extract_results(self) + def move_failed(self, job): self.run_output.move_failed(job.output) @@ -173,6 +178,22 @@ class ExecutionContext(object): classifiers) self.output.add_metric(name, value, units, lower_is_better, classifiers) + def get_artifact(self, name): + try: + return self.output.get_artifact(name) + except HostError: + if not self.current_job: + raise + return self.run_output.get_artifact(name) + + def get_artifact_path(self, name): + try: + return self.output.get_artifact_path(name) + except HostError: + if not self.current_job: + raise + return self.run_output.get_artifact_path(name) + def add_artifact(self, name, path, kind, description=None, classifiers=None): self.output.add_artifact(name, path, kind, description, classifiers) diff --git a/wa/framework/getters.py b/wa/framework/getters.py index 9ce5f94e..651ca5a2 100644 --- a/wa/framework/getters.py +++ b/wa/framework/getters.py @@ -52,7 +52,7 @@ def get_by_extension(path, ext): found = [] for entry in os.listdir(path): - entry_ext = os.path.splitext(entry) + entry_ext = os.path.splitext(entry)[1] if entry_ext == ext: found.append(os.path.join(path, entry)) return found @@ -112,7 +112,8 @@ class UserDirectory(ResourceGetter): def get(self, resource): basepath = settings.dependencies_directory - return get_from_location(basepath, resource) + directory = _d(os.path.join(basepath, resource.owner.name)) + return get_from_location(directory, resource) class Http(ResourceGetter): @@ -318,7 +319,9 @@ class Filer(ResourceGetter): result = get_from_location(local_path, resource) if result: return result - if remote_path: + if not os.path.exists(local_path): + return None + if os.path.exists(remote_path): # Didn't find it cached locally; now check the remoted result = get_from_location(remote_path, resource) if not result: diff --git a/wa/framework/instrumentation.py b/wa/framework/instrumentation.py index f96c0cec..3fe67de3 100644 --- a/wa/framework/instrumentation.py +++ b/wa/framework/instrumentation.py @@ -125,8 +125,8 @@ SIGNAL_MAP = OrderedDict([ ('setup', signal.BEFORE_WORKLOAD_SETUP), ('start', signal.BEFORE_WORKLOAD_EXECUTION), ('stop', signal.AFTER_WORKLOAD_EXECUTION), - ('process_workload_result', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE), - ('update_result', signal.AFTER_WORKLOAD_RESULT_UPDATE), + ('process_workload_output', signal.SUCCESSFUL_WORKLOAD_OUTPUT_UPDATE), + ('update_result', signal.AFTER_WORKLOAD_OUTPUT_UPDATE), ('teardown', signal.AFTER_WORKLOAD_TEARDOWN), ('finalize', signal.RUN_FINALIZED), diff --git a/wa/framework/job.py b/wa/framework/job.py index 9b7bc7da..880cfff4 100644 --- a/wa/framework/job.py +++ b/wa/framework/job.py @@ -68,8 +68,11 @@ class Job(object): def process_output(self, context): self.logger.info('Processing output for job {}'.format(self.id)) - with signal.wrap('WORKLOAD_RESULT_UPDATE', self, context): - self.workload.update_result(context) + with signal.wrap('WORKLOAD_RESULT_EXTRACTION', self, context): + self.workload.extract_results(context) + context.extract_results() + with signal.wrap('WORKLOAD_OUTPUT_UPDATE', self, context): + self.workload.update_output(context) def teardown(self, context): self.logger.info('Tearing down job {}'.format(self.id)) diff --git a/wa/framework/output.py b/wa/framework/output.py index daf3d83a..53c9fd3d 100644 --- a/wa/framework/output.py +++ b/wa/framework/output.py @@ -84,6 +84,13 @@ class Output(object): def add_event(self, message): self.result.add_event(message) + def get_artifact(self, name): + return self.result.get_artifact(name) + + def get_artifact_path(self, name): + artifact = self.get_artifact(name) + return self.get_path(artifact.path) + class RunOutput(Output): @@ -234,6 +241,12 @@ class Result(object): def add_event(self, message): self.events.append(Event(message)) + def get_artifact(self, name): + for artifact in self.artifacts: + if artifact.name == name: + return artifact + raise HostError('Artifact "{}" not found'.format(name)) + def to_pod(self): return dict( status=str(self.status), @@ -465,7 +478,7 @@ def init_job_output(run_output, job): path = os.path.join(run_output.basepath, output_name) ensure_directory_exists(path) write_pod(Result().to_pod(), os.path.join(path, 'result.json')) - job_output = JobOutput(path, job.id, job.iteration, job.label, job.retries) + job_output = JobOutput(path, job.id, job.label, job.iteration, job.retries) job_output.status = job.status run_output.jobs.append(job_output) return job_output diff --git a/wa/framework/resource.py b/wa/framework/resource.py index 2ce9de7f..44745386 100644 --- a/wa/framework/resource.py +++ b/wa/framework/resource.py @@ -75,7 +75,7 @@ class Resource(object): raise NotImplementedError() def __str__(self): - return '<{}\'s {}>'.format(self.owner, self.name) + return '<{}\'s {}>'.format(self.owner, self.kind) class File(Resource): diff --git a/wa/framework/signal.py b/wa/framework/signal.py index 09fe7b4e..20c6a0b2 100644 --- a/wa/framework/signal.py +++ b/wa/framework/signal.py @@ -97,7 +97,8 @@ JOB_FINALIZED = Signal('job-finalized') # Signals associated with particular stages of workload execution -BEFORE_WORKLOAD_INITIALIZED = Signal('before-workload-initialized', invert_priority=True) +BEFORE_WORKLOAD_INITIALIZED = Signal('before-workload-initialized', + invert_priority=True) SUCCESSFUL_WORKLOAD_INITIALIZED = Signal('successful-workload-initialized') AFTER_WORKLOAD_INITIALIZED = Signal('after-workload-initialized') @@ -109,9 +110,15 @@ BEFORE_WORKLOAD_EXECUTION = Signal('before-workload-execution', invert_priority= SUCCESSFUL_WORKLOAD_EXECUTION = Signal('successful-workload-execution') AFTER_WORKLOAD_EXECUTION = Signal('after-workload-execution') -BEFORE_WORKLOAD_RESULT_UPDATE = Signal('before-workload-result-update', invert_priority=True) -SUCCESSFUL_WORKLOAD_RESULT_UPDATE = Signal('successful-workload-result-update') -AFTER_WORKLOAD_RESULT_UPDATE = Signal('after-workload-result-update') +BEFORE_WORKLOAD_RESULT_EXTRACTION = Signal('before-workload-result-exptracton', + invert_priority=True) +SUCCESSFUL_WORKLOAD_RESULT_EXTRACTION = Signal('successful-workload-result-exptracton') +AFTER_WORKLOAD_RESULT_EXTRACTION = Signal('after-workload-result-exptracton') + +BEFORE_WORKLOAD_OUTPUT_UPDATE = Signal('before-workload-output-update', + invert_priority=True) +SUCCESSFUL_WORKLOAD_OUTPUT_UPDATE = Signal('successful-workload-output-update') +AFTER_WORKLOAD_OUTPUT_UPDATE = Signal('after-workload-output-update') BEFORE_WORKLOAD_TEARDOWN = Signal('before-workload-teardown', invert_priority=True) SUCCESSFUL_WORKLOAD_TEARDOWN = Signal('successful-workload-teardown') diff --git a/wa/framework/target/assistant.py b/wa/framework/target/assistant.py new file mode 100644 index 00000000..99411059 --- /dev/null +++ b/wa/framework/target/assistant.py @@ -0,0 +1,144 @@ +import logging +import os +import shutil +import sys +import tempfile +import threading +import time + +from wa import Parameter +from wa.framework.exception import WorkerThreadError + + +class LinuxAssistant(object): + + parameters = [] + + def __init__(self, target): + self.target = target + + def start(self): + pass + + def extract_results(self, context): + pass + + def stop(self): + pass + + +class AndroidAssistant(object): + + parameters = [ + Parameter('logcat_poll_period', kind=int, + constraint=lambda x: x > 0, + description=""" + Polling period for logcat in seconds. If not specified, + no polling will be used. + + Logcat buffer on android is of limited size and it cannot be + adjusted at run time. Depending on the amount of logging activity, + the buffer may not be enought to capture comlete trace for a + workload execution. For those situations, logcat may be polled + periodically during the course of the run and stored in a + temporary locaiton on the host. Setting the value of the poll + period enables this behavior. + """), + ] + + def __init__(self, target, logcat_poll_period=None): + self.target = target + if logcat_poll_period: + self.logcat_poller = LogcatPoller(target, logcat_poll_period) + else: + self.logcat_poller = None + + def start(self): + if self.logcat_poller: + self.logcat_poller.start() + + def stop(self): + if self.logcat_poller: + self.logcat_poller.stop() + + def extract_results(self, context): + logcat_file = os.path.join(context.output_directory, 'logcat.log') + self.dump_logcat(logcat_file) + context.add_artifact('logcat', logcat_file, kind='log') + self.clear_logcat() + + def dump_logcat(self, outfile): + if self.logcat_poller: + self.logcat_poller.write_log(outfile) + else: + self.target.dump_logcat(outfile) + + def clear_logcat(self): + if self.logcat_poller: + self.logcat_poller.clear_buffer() + + +class LogcatPoller(threading.Thread): + + def __init__(self, target, period=60, timeout=30): + super(LogcatPoller, self).__init__() + self.target = target + self.logger = logging.getLogger('logcat') + self.period = period + self.timeout = timeout + self.stop_signal = threading.Event() + self.lock = threading.Lock() + self.buffer_file = tempfile.mktemp() + self.last_poll = 0 + self.daemon = True + self.exc = None + + def start(self): + self.logger.debug('starting polling') + try: + while True: + if self.stop_signal.is_set(): + break + with self.lock: + current_time = time.time() + if (current_time - self.last_poll) >= self.period: + self.poll() + time.sleep(0.5) + except Exception: # pylint: disable=W0703 + self.exc = WorkerThreadError(self.name, sys.exc_info()) + self.logger.debug('polling stopped') + + def stop(self): + self.logger.debug('Stopping logcat polling') + self.stop_signal.set() + self.join(self.timeout) + if self.is_alive(): + self.logger.error('Could not join logcat poller thread.') + if self.exc: + raise self.exc # pylint: disable=E0702 + + def clear_buffer(self): + self.logger.debug('clearing logcat buffer') + with self.lock: + self.target.clear_logcat() + with open(self.buffer_file, 'w') as _: # NOQA + pass + + def write_log(self, outfile): + with self.lock: + self.poll() + if os.path.isfile(self.buffer_file): + shutil.copy(self.buffer_file, outfile) + else: # there was no logcat trace at this time + with open(outfile, 'w') as _: # NOQA + pass + + def close(self): + self.logger.debug('closing poller') + if os.path.isfile(self.buffer_file): + os.remove(self.buffer_file) + + def poll(self): + self.last_poll = time.time() + self.target.dump_logcat(self.buffer_file, append=True, timeout=self.timeout) + self.target.clear_logcat() diff --git a/wa/framework/target/descriptor.py b/wa/framework/target/descriptor.py index 717df67d..9e01ebac 100644 --- a/wa/framework/target/descriptor.py +++ b/wa/framework/target/descriptor.py @@ -7,6 +7,7 @@ from devlib import (LinuxTarget, AndroidTarget, LocalLinuxTarget, from wa.framework import pluginloader from wa.framework.exception import PluginLoaderError from wa.framework.plugin import Plugin, Parameter +from wa.framework.target.assistant import LinuxAssistant, AndroidAssistant from wa.utils.types import list_of_strings, list_of_ints from wa.utils.misc import isiterable @@ -28,6 +29,7 @@ def instantiate_target(tdesc, params, connect=None): target_params = {p.name: p for p in tdesc.target_params} platform_params = {p.name: p for p in tdesc.platform_params} conn_params = {p.name: p for p in tdesc.conn_params} + assistant_params = {p.name: p for p in tdesc.assistant_params} tp, pp, cp = {}, {}, {} @@ -38,6 +40,8 @@ def instantiate_target(tdesc, params, connect=None): pp[name] = value elif name in conn_params: cp[name] = value + elif name in assistant_params: + pass else: msg = 'Unexpected parameter for {}: {}' raise ValueError(msg.format(tdesc.name, name)) @@ -53,17 +57,27 @@ def instantiate_target(tdesc, params, connect=None): return tdesc.target(**tp) +def instantiate_assistant(tdesc, params, target): + assistant_params = {} + for param in tdesc.assistant_params: + if param.name in params: + assistant_params[param.name] = params[param.name] + return tdesc.assistant(target, **assistant_params) + + class TargetDescription(object): def __init__(self, name, source, description=None, target=None, platform=None, - conn=None, target_params=None, platform_params=None, - conn_params=None): + conn=None, assistant=None, target_params=None, platform_params=None, + conn_params=None, assistant_params=None): self.name = name self.source = source self.description = description self.target = target self.platform = platform self.connection = conn + self.assistant = assistant + self.assistant_params = assistant_params self._set('target_params', target_params) self._set('platform_params', platform_params) self._set('conn_params', conn_params) @@ -218,7 +232,7 @@ GEM5_PLATFORM_PARAMS = [ '''), ] -# name --> (target_class, params_list, defaults) +# name --> (target_class, params_list, defaults, assistant_class) TARGETS = { 'linux': (LinuxTarget, COMMON_TARGET_PARAMS, None), 'android': (AndroidTarget, COMMON_TARGET_PARAMS + @@ -230,6 +244,13 @@ TARGETS = { 'local': (LocalLinuxTarget, COMMON_TARGET_PARAMS, None), } +# name --> assistant +ASSISTANTS = { + 'linux': LinuxAssistant, + 'android': AndroidAssistant, + 'local': LinuxAssistant, +} + # name --> (platform_class, params_list, defaults) PLATFORMS = { 'generic': (Platform, COMMON_PLATFORM_PARAMS, None), @@ -267,6 +288,7 @@ class DefaultTargetDescriptor(TargetDescriptor): result = [] for target_name, target_tuple in TARGETS.iteritems(): target, target_params = self._get_item(target_tuple) + assistant = ASSISTANTS[target_name] for platform_name, platform_tuple in PLATFORMS.iteritems(): platform, platform_params = self._get_item(platform_tuple) @@ -274,8 +296,10 @@ class DefaultTargetDescriptor(TargetDescriptor): td = TargetDescription(name, self) td.target = target td.platform = platform + td.assistant = assistant td.target_params = target_params td.platform_params = platform_params + td.assistant_params = assistant.parameters result.append(td) return result diff --git a/wa/framework/target/manager.py b/wa/framework/target/manager.py index 043dbcb4..f1f233c9 100644 --- a/wa/framework/target/manager.py +++ b/wa/framework/target/manager.py @@ -10,7 +10,8 @@ from wa.framework import signal from wa.framework.exception import WorkerThreadError, ConfigError from wa.framework.plugin import Parameter from wa.framework.target.descriptor import (get_target_descriptions, - instantiate_target) + instantiate_target, + instantiate_assistant) from wa.framework.target.info import TargetInfo from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig, HotplugRuntimeConfig, @@ -62,7 +63,6 @@ class TargetManager(object): self.info = TargetInfo() self._init_target() - self._init_assistant() self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] def finalize(self): @@ -83,6 +83,15 @@ class TargetManager(object): if any(parameter in name for parameter in cfg.supported_parameters): cfg.add(name, self.parameters.pop(name)) + def start(self): + self.assistant.start() + + def stop(self): + self.assistant.stop() + + def extract_results(self, context): + self.assistant.extract_results(context) + @memoized def get_target_info(self): return TargetInfo(self.target) @@ -107,238 +116,12 @@ class TargetManager(object): if self.target_name not in target_map: raise ValueError('Unknown Target: {}'.format(self.target_name)) tdesc = target_map[self.target_name] + self.target = instantiate_target(tdesc, self.parameters, connect=False) + with signal.wrap('TARGET_CONNECT'): self.target.connect() self.logger.info('Setting up target') self.target.setup() - def _init_assistant(self): - # Create a corresponding target and target-assistant to help with - # platformy stuff? - if self.target.os == 'android': - self.assistant = AndroidAssistant(self.target) - elif self.target.os == 'linux': - self.assistant = LinuxAssistant(self.target) # pylint: disable=redefined-variable-type - else: - raise ValueError('Unknown Target OS: {}'.format(self.target.os)) - - -class LinuxAssistant(object): - - name = 'linux-assistant' - - description = """ - Performs configuration, instrumentation, etc. during runs on Linux targets. - """ - - def __init__(self, target, **kwargs): - self.target = target - # parameters = [ - - # Parameter('disconnect', kind=bool, default=False, - # description=""" - # Specifies whether the target should be disconnected from - # at the end of the run. - # """), - # ] - - # runtime_config_cls = [ - # # order matters - # SysfileValuesRuntimeConfig, - # HotplugRuntimeConfig, - # CpufreqRuntimeConfig, - # CpuidleRuntimeConfig, - # ] - - # def __init__(self, target, context, **kwargs): - # # super(LinuxTargetManager, self).__init__(target, context, **kwargs) - # self.target = target - # self.context = context - # self.info = TargetInfo() - # self.runtime_configs = [cls(target) for cls in self.runtime_config_cls] - - # def __init__(self): - # # super(LinuxTargetManager, self).__init__(target, context, **kwargs) - # self.target = target - # self.info = TargetInfo() - # self.parameters = parameters - - # self.info = TargetInfo() - # self.runtime_configs = [cls(target) for cls in self.runtime_config_cls] - - # def initialize(self): - # # self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] - # # if self.parameters: - # self.logger.info('Connecting to the device') - # with signal.wrap('TARGET_CONNECT'): - # self.target.connect() - # self.info.load(self.target) - # # info_file = os.path.join(self.context.info_directory, 'target.json') - # # with open(info_file, 'w') as wfh: - # # json.dump(self.info.to_pod(), wfh) - - # def finalize(self, runner): - # self.logger.info('Disconnecting from the device') - # if self.disconnect: - # with signal.wrap('TARGET_DISCONNECT'): - # self.target.disconnect() - - # def _add_parameters(self): - # for name, value in self.parameters.iteritems(): - # self.add_parameter(name, value) - - # def validate_runtime_parameters(self, parameters): - # self.clear() - # for name, value in parameters.iteritems(): - # self.add_parameter(name, value) - # self.validate_parameters() - - # def set_runtime_parameters(self, parameters): - # self.clear() - # for name, value in parameters.iteritems(): - # self.add_parameter(name, value) - # self.set_parameters() - - # def clear_parameters(self): - # for cfg in self.runtime_configs: - # cfg.clear() - - # def add_parameter(self, name, value): - # for cfg in self.runtime_configs: - # if name in cfg.supported_parameters: - # cfg.add(name, value) - # return - # raise ConfigError('Unexpected runtime parameter "{}".'.format(name)) - - # def validate_parameters(self): - # for cfg in self.runtime_configs: - # cfg.validate() - - # def set_parameters(self): - # for cfg in self.runtime_configs: - # cfg.set() - - -class AndroidAssistant(LinuxAssistant): - - name = 'android-assistant' - description = """ - Extends ``LinuxTargetManager`` with Android-specific operations. - """ - - parameters = [ - Parameter('logcat_poll_period', kind=int, - description=""" - If specified, logcat will cached in a temporary file on the - host every ``logcat_poll_period`` seconds. This is useful for - longer job executions, where on-device logcat buffer may not be - big enough to capture output for the entire execution. - """), - ] - - def __init__(self, target, **kwargs): - super(AndroidAssistant, self).__init__(target) - self.logcat_poll_period = kwargs.get('logcat_poll_period', None) - if self.logcat_poll_period: - self.logcat_poller = LogcatPoller(target, self.logcat_poll_period) - else: - self.logcat_poller = None - - # def __init__(self, target, context, **kwargs): - # super(AndroidAssistant, self).__init__(target, context, **kwargs) - # self.logcat_poll_period = kwargs.get('logcat_poll_period', None) - # if self.logcat_poll_period: - # self.logcat_poller = LogcatPoller(target, self.logcat_poll_period) - # else: - # self.logcat_poller = None - - # def next_job(self, job): - # super(AndroidAssistant, self).next_job(job) - # if self.logcat_poller: - # self.logcat_poller.start() - - # def job_done(self, job): - # super(AndroidAssistant, self).job_done(job) - # if self.logcat_poller: - # self.logcat_poller.stop() - # outfile = os.path.join(self.context.output_directory, 'logcat.log') - # self.logger.debug('Dumping logcat to {}'.format(outfile)) - # self.dump_logcat(outfile) - # self.clear() - - def dump_logcat(self, outfile): - if self.logcat_poller: - self.logcat_poller.write_log(outfile) - else: - self.target.dump_logcat(outfile) - - def clear_logcat(self): - if self.logcat_poller: - self.logcat_poller.clear_buffer() - - -class LogcatPoller(threading.Thread): - - def __init__(self, target, period=60, timeout=30): - super(LogcatPoller, self).__init__() - self.target = target - self.logger = logging.getLogger('logcat') - self.period = period - self.timeout = timeout - self.stop_signal = threading.Event() - self.lock = threading.Lock() - self.buffer_file = tempfile.mktemp() - self.last_poll = 0 - self.daemon = True - self.exc = None - - def start(self): - self.logger.debug('starting polling') - try: - while True: - if self.stop_signal.is_set(): - break - with self.lock: - current_time = time.time() - if (current_time - self.last_poll) >= self.period: - self.poll() - time.sleep(0.5) - except Exception: # pylint: disable=W0703 - self.exc = WorkerThreadError(self.name, sys.exc_info()) - self.logger.debug('polling stopped') - - def stop(self): - self.logger.debug('Stopping logcat polling') - self.stop_signal.set() - self.join(self.timeout) - if self.is_alive(): - self.logger.error('Could not join logcat poller thread.') - if self.exc: - raise self.exc # pylint: disable=E0702 - - def clear_buffer(self): - self.logger.debug('clearing logcat buffer') - with self.lock: - self.target.clear_logcat() - with open(self.buffer_file, 'w') as _: # NOQA - pass - - def write_log(self, outfile): - with self.lock: - self.poll() - if os.path.isfile(self.buffer_file): - shutil.copy(self.buffer_file, outfile) - else: # there was no logcat trace at this time - with open(outfile, 'w') as _: # NOQA - pass - - def close(self): - self.logger.debug('closing poller') - if os.path.isfile(self.buffer_file): - os.remove(self.buffer_file) - - def poll(self): - self.last_poll = time.time() - self.target.dump_logcat(self.buffer_file, append=True, timeout=self.timeout) - self.target.clear_logcat() + self.assistant = instantiate_assistant(tdesc, self.parameters, self.target) diff --git a/wa/framework/uiauto/BaseUiAutomation.class b/wa/framework/uiauto/BaseUiAutomation.class new file mode 100644 index 00000000..942f7ca3 Binary files /dev/null and b/wa/framework/uiauto/BaseUiAutomation.class differ diff --git a/wa/framework/uiauto/build.sh b/wa/framework/uiauto/build.sh new file mode 100755 index 00000000..1beddd6a --- /dev/null +++ b/wa/framework/uiauto/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + + +ant build + +cp bin/classes/com/arm/wa/uiauto/BaseUiAutomation.class . diff --git a/wa/framework/uiauto/build.xml b/wa/framework/uiauto/build.xml new file mode 100644 index 00000000..86b8aa0f --- /dev/null +++ b/wa/framework/uiauto/build.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/wa/framework/uiauto/project.properties b/wa/framework/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wa/framework/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java b/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java new file mode 100644 index 00000000..fc0faded --- /dev/null +++ b/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java @@ -0,0 +1,124 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wa.uiauto; + +import java.io.File; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.concurrent.TimeoutException; + +import android.app.Activity; +import android.os.Bundle; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +public class BaseUiAutomation extends UiAutomatorTestCase { + + public void setup() throws Exception { + } + + public void runWorkload() throws Exception { + } + + public void extractResults() throws Exception { + } + + public void teardown() throws Exception { + } + + public void sleep(int second) { + super.sleep(second * 1000); + } + + public boolean takeScreenshot(String name) { + Bundle params = getParams(); + String png_dir = params.getString("workdir"); + + try { + return getUiDevice().takeScreenshot(new File(png_dir, name + ".png")); + } catch(NoSuchMethodError e) { + return true; + } + } + + public void waitText(String text) throws UiObjectNotFoundException { + waitText(text, 600); + } + + public void waitText(String text, int second) throws UiObjectNotFoundException { + UiSelector selector = new UiSelector(); + UiObject text_obj = new UiObject(selector.text(text) + .className("android.widget.TextView")); + waitObject(text_obj, second); + } + + public void waitObject(UiObject obj) throws UiObjectNotFoundException { + waitObject(obj, 600); + } + + public void waitObject(UiObject obj, int second) throws UiObjectNotFoundException { + if (! obj.waitForExists(second * 1000)){ + throw new UiObjectNotFoundException("UiObject is not found: " + + obj.getSelector().toString()); + } + } + + public boolean waitUntilNoObject(UiObject obj, int second) { + return obj.waitUntilGone(second * 1000); + } + + public void clearLogcat() throws Exception { + Runtime.getRuntime().exec("logcat -c"); + } + + public void waitForLogcatText(String searchText, long timeout) throws Exception { + long startTime = System.currentTimeMillis(); + Process process = Runtime.getRuntime().exec("logcat"); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + + long currentTime = System.currentTimeMillis(); + boolean found = false; + while ((currentTime - startTime) < timeout){ + sleep(2); // poll every two seconds + + while((line=reader.readLine())!=null) { + if (line.contains(searchText)) { + found = true; + break; + } + } + + if (found) { + break; + } + currentTime = System.currentTimeMillis(); + } + + process.destroy(); + + if ((currentTime - startTime) >= timeout) { + throw new TimeoutException("Timed out waiting for Logcat text \"%s\"".format(searchText)); + } + } +} + diff --git a/wa/framework/workload.py b/wa/framework/workload.py index 850bceca..b2f565e3 100644 --- a/wa/framework/workload.py +++ b/wa/framework/workload.py @@ -13,9 +13,11 @@ # limitations under the License. # import logging +import os +import time from wa.framework.plugin import TargetedPlugin -from wa.framework.resource import JarFile, ReventFile, NO_ONE +from wa.framework.resource import ApkFile, JarFile, ReventFile, NO_ONE from wa.framework.exception import WorkloadError from devlib.utils.android import ApkInfo @@ -66,10 +68,16 @@ class Workload(TargetedPlugin): """ pass - def update_result(self, context): + def extract_results(self, context): """ - Update the result within the specified execution context with the - metrics form this workload iteration. + Extract results on the target + """ + pass + + def update_output(self, context): + """ + Update the output within the specified execution context with the + metrics and artifacts form this workload iteration. """ pass @@ -85,53 +93,117 @@ class Workload(TargetedPlugin): return ''.format(self.name) -class UiAutomatorGUI(object): +class ApkUiautoWorkload(Workload): + + platform = 'android' - def __init__(self, target, package='', klass='UiAutomation', - method='runUiAutoamtion'): - self.target = target - self.uiauto_package = package - self.uiauto_class = klass - self.uiauto_method = method - self.uiauto_file = None - self.target_uiauto_file = None - self.command = None - self.uiauto_params = {} + def __init__(self, target, **kwargs): + super(ApkUiautoWorkload, self).__init__(target, **kwargs) + self.apk = ApkHander(self) + self.gui = UiAutomatorGUI(self) def init_resources(self, context): - self.uiauto_file = context.resolver.get(JarFile(self)) - self.target_uiauto_file = self.target.path.join(self.target.working_directory, - os.path.basename(self.uiauto_file)) - if not self.uiauto_package: - self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] + self.apk.init_resources(context.resolver) + self.gui.init_resources(context.resolver) + self.gui.init_commands() - def validate(self): - if not self.uiauto_file: - raise WorkloadError('No UI automation JAR file found for workload {}.'.format(self.name)) - if not self.uiauto_package: - raise WorkloadError('No UI automation package specified for workload {}.'.format(self.name)) + def initialize(self, context): + self.gui.deploy() def setup(self, context): - method_string = '{}.{}#{}'.format(self.uiauto_package, self.uiauto_class, self.uiauto_method) + self.apk.setup(context) + self.gui.setup() + + def run(self, context): + self.gui.run() + + def extract_results(self, context): + self.gui.extract_results() + + def teardown(self, context): + self.gui.teardown() + self.apk.teardown() + + def finalize(self, context): + self.gui.remove() + + +class UiAutomatorGUI(object): + + stages = ['setup', 'runWorkload', 'extractResults', 'teardown'] + + def __init__(self, owner, package=None, klass='UiAutomation', timeout=600): + self.owner = owner + self.target = self.owner.target + self.uiauto_package = package + self.uiauto_class = klass + self.timeout = timeout + self.logger = logging.getLogger('gui') + self.jar_file = None + self.target_jar_file = None + self.commands = {} + self.uiauto_params = {} + + def init_resources(self, resolver): + self.jar_file = resolver.get(JarFile(self.owner)) + jar_name = os.path.basename(self.jar_file) + self.target_jar_file = self.target.get_workpath(jar_name) + if not self.uiauto_package: + package = os.path.splitext(os.path.basename(self.jar_file))[0] + self.uiauto_package = package + + def init_commands(self): params_dict = self.uiauto_params params_dict['workdir'] = self.target.working_directory params = '' for k, v in self.uiauto_params.iteritems(): params += ' -e {} {}'.format(k, v) - self.command = 'uiautomator runtest {}{} -c {}'.format(self.target_uiauto_file, params, method_string) - self.target.push_file(self.uiauto_file, self.target_uiauto_file) - self.target.killall('uiautomator') - def run(self, context): - result = self.target.execute(self.command, self.run_timeout) + for stage in self.stages: + method_string = '{}.{}#{}'.format(self.uiauto_package, + self.uiauto_class, + stage) + cmd_template = 'uiautomator runtest {}{} -c {}' + self.commands[stage] = cmd_template.format(self.target_jar_file, + params, method_string) + + def deploy(self): + self.target.push(self.jar_file, self.target_jar_file) + + def set(self, name, value): + self.uiauto_params[name] = value + + def setup(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self.target.killall('uiautomator') + self._execute('setup', timeout or self.timeout) + + def run(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('runWorkload', timeout or self.timeout) + + def extract_results(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('extractResults', timeout or self.timeout) + + def teardown(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('teardown', timeout or self.timeout) + + def remove(self): + self.target.remove(self.target_jar_file) + + def _execute(self, stage, timeout): + result = self.target.execute(self.commands[stage], timeout) if 'FAILURE' in result: raise WorkloadError(result) else: self.logger.debug(result) - time.sleep(DELAY) - - def teardown(self, context): - self.target.delete_file(self.target_uiauto_file) + time.sleep(2) class ReventGUI(object): @@ -191,21 +263,27 @@ class ReventGUI(object): class ApkHander(object): - def __init__(self, owner, target, view, install_timeout=300, version=None, + def __init__(self, owner, install_timeout=300, version=None, variant=None, strict=True, force_install=False, uninstall=False): self.logger = logging.getLogger('apk') self.owner = owner - self.target = target + self.target = self.owner.target + self.install_timeout = install_timeout self.version = version + self.variant = variant + self.strict = strict + self.force_install = force_install + self.uninstall = uninstall self.apk_file = None self.apk_info = None self.apk_version = None self.logcat_log = None - def init_resources(self, context): - self.apk_file = context.resolver.get(ApkFile(self.owner), - version=self.version, - strict=strict) + def init_resources(self, resolver): + self.apk_file = resolver.get(ApkFile(self.owner, + variant=self.variant, + version=self.version), + strict=self.strict) self.apk_info = ApkInfo(self.apk_file) def setup(self, context): @@ -220,16 +298,17 @@ class ApkHander(object): self.initialize_with_host_apk(context, installed_version) else: if not installed_version: - message = '''{} not found found on the device and check_apk is set to "False" - so host version was not checked.''' - raise WorkloadError(message.format(self.package)) + message = '{} not found found on the device and check_apk is set '\ + 'to "False" so host version was not checked.' + raise WorkloadError(message.format(self.apk_info.package)) message = 'Version {} installed on device; skipping host APK check.' self.logger.debug(message.format(installed_version)) self.reset(context) self.version = installed_version def initialize_with_host_apk(self, context, installed_version): - if installed_version != self.apk_file.version_name: + host_version = self.apk_info.version_name + if installed_version != host_version: if installed_version: message = '{} host version: {}, device version: {}; re-installing...' self.logger.debug(message.format(os.path.basename(self.apk_file), @@ -245,42 +324,38 @@ class ApkHander(object): host_version)) if self.force_install: if installed_version: - self.device.uninstall(self.package) + self.target.uninstall_package(self.apk_info.package) self.install_apk(context) else: self.reset(context) self.apk_version = host_version def start_activity(self): - output = self.device.execute('am start -W -n {}/{}'.format(self.package, self.activity)) + cmd = 'am start -W -n {}/{}' + output = self.target.execute(cmd.format(self.apk_info.package, + self.apk_info.activity)) if 'Error:' in output: - self.device.execute('am force-stop {}'.format(self.package)) # this will dismiss any erro dialogs + # this will dismiss any error dialogs + self.target.execute('am force-stop {}'.format(self.apk_info.package)) raise WorkloadError(output) self.logger.debug(output) def reset(self, context): # pylint: disable=W0613 - self.device.execute('am force-stop {}'.format(self.package)) - self.device.execute('pm clear {}'.format(self.package)) + self.target.execute('am force-stop {}'.format(self.apk_info.package)) + self.target.execute('pm clear {}'.format(self.apk_info.package)) def install_apk(self, context): - output = self.device.install(self.apk_file, self.install_timeout) + output = self.target.install_apk(self.apk_file, self.install_timeout) if 'Failure' in output: if 'ALREADY_EXISTS' in output: - self.logger.warn('Using already installed APK (did not unistall properly?)') + msg = 'Using already installed APK (did not unistall properly?)' + self.logger.warn(msg) else: raise WorkloadError(output) else: self.logger.debug(output) - def update_result(self, context): - self.logcat_log = os.path.join(context.output_directory, 'logcat.log') - self.device.dump_logcat(self.logcat_log) - context.add_iteration_artifact(name='logcat', - path='logcat.log', - kind='log', - description='Logact dump for the run.') - - def teardown(self, context): - self.device.execute('am force-stop {}'.format(self.package)) - if self.uninstall_apk: - self.device.uninstall(self.package) + def teardown(self): + self.target.execute('am force-stop {}'.format(self.apk_info.package)) + if self.uninstall: + self.target.uninstall_package(self.apk_info.package) diff --git a/wa/workloads/benchmarkpi/__init__.py b/wa/workloads/benchmarkpi/__init__.py new file mode 100644 index 00000000..d731305e --- /dev/null +++ b/wa/workloads/benchmarkpi/__init__.py @@ -0,0 +1,62 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import re + +from wa import ApkUiautoWorkload + + +class BenchmarkPi(ApkUiautoWorkload): + + name = 'benchmarkpi' + description = """ + Measures the time the target device takes to run and complete the Pi + calculation algorithm. + + http://androidbenchmark.com/howitworks.php + + from the website: + + The whole idea behind this application is to use the same Pi calculation + algorithm on every Android Device and check how fast that proccess is. + Better calculation times, conclude to faster Android devices. This way you + can also check how lightweight your custom made Android build is. Or not. + + As Pi is an irrational number, Benchmark Pi does not calculate the actual Pi + number, but an approximation near the first digits of Pi over the same + calculation circles the algorithms needs. + + So, the number you are getting in miliseconds is the time your mobile device + takes to run and complete the Pi calculation algorithm resulting in a + approximation of the first Pi digits. + """ + package = 'gr.androiddev.BenchmarkPi' + activity = '.BenchmarkPi' + + regex = re.compile('You calculated Pi in ([0-9]+)') + + def update_output(self, context): + super(BenchmarkPi, self).update_output(context) + logcat_file = context.get_artifact_path('logcat') + with open(logcat_file) as fh: + for line in fh: + match = self.regex.search(line) + if match: + result = int(match.group(1)) + + if result is not None: + context.add_metric('pi calculation', result, + 'milliseconds', lower_is_better=True) diff --git a/wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar b/wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar new file mode 100644 index 00000000..e597b3b5 Binary files /dev/null and b/wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar differ diff --git a/wa/workloads/benchmarkpi/uiauto/build.sh b/wa/workloads/benchmarkpi/uiauto/build.sh new file mode 100755 index 00000000..3c5f5ba8 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +class_dir=bin/classes/com/arm/wa/uiauto +base_class=`python -c "import os, wa; print os.path.join(os.path.dirname(wa.__file__), 'framework', 'uiauto', 'BaseUiAutomation.class')"` +mkdir -p $class_dir +cp $base_class $class_dir + +ant build + +if [[ -f bin/com.arm.wa.uiauto.benchmarkpi.jar ]]; then + cp bin/com.arm.wa.uiauto.benchmarkpi.jar .. +fi diff --git a/wa/workloads/benchmarkpi/uiauto/build.xml b/wa/workloads/benchmarkpi/uiauto/build.xml new file mode 100644 index 00000000..3f8f1495 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/build.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/wa/workloads/benchmarkpi/uiauto/project.properties b/wa/workloads/benchmarkpi/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java b/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java new file mode 100644 index 00000000..097578f2 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java @@ -0,0 +1,62 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wa.uiauto.benchmarkpi; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wa.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "benchmarkpi"; + + public void runWorkload() throws Exception { + startTest(); + waitForResults(); + } + + public void extractResults() throws Exception { + UiSelector selector = new UiSelector(); + UiObject resultsText = new UiObject(selector.textContains("You calculated Pi in") + .className("android.widget.TextView")); + Log.v(TAG, resultsText.getText()); + } + + public void startTest() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject benchButton = new UiObject(selector.text("Benchmark my Android!") + .className("android.widget.Button")); + benchButton.click(); + } + + public void waitForResults() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject submitButton = new UiObject(selector.text("Submit") + .className("android.widget.Button")); + submitButton.waitForExists(10 * 1000); + } + +} diff --git a/wa/workloads/dhrystone/__init__.py b/wa/workloads/dhrystone/__init__.py index 106d0483..369e84bf 100644 --- a/wa/workloads/dhrystone/__init__.py +++ b/wa/workloads/dhrystone/__init__.py @@ -105,12 +105,13 @@ class Dhrystone(Workload): self.target.killall('dhrystone') raise - def update_result(self, context): + def extract_results(self, context): outfile = os.path.join(context.output_directory, 'dhrystone.output') with open(outfile, 'w') as wfh: wfh.write(self.output) context.add_artifact('dhrystone-output', outfile, 'raw', "dhrystone's stdout") + def update_output(self, context): score_count = 0 dmips_count = 0 total_score = 0