From 0032e347fe9b6f2678f86caa9e235592b854f41e Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Mon, 27 Mar 2017 17:31:44 +0100 Subject: [PATCH 1/5] Implemeting target assistants - Workload's update_result stage has now been broken up into two parts: extract_results and update_output. This is to allow the assistant to pull output from the target in between the two stages. - Updated assistant implementations for Linux and Android targets from the exisiting code. - Extended target descriptor code to handle assistants and their parameters as well. - Updated the target manager to actually make use of the assistants. --- wa/framework/execution.py | 5 + wa/framework/instrumentation.py | 4 +- wa/framework/job.py | 7 +- wa/framework/signal.py | 15 +- wa/framework/target/assistant.py | 144 +++++++++++++++++ wa/framework/target/descriptor.py | 30 +++- wa/framework/target/manager.py | 245 ++--------------------------- wa/framework/workload.py | 12 +- wa/workloads/dhrystone/__init__.py | 3 +- 9 files changed, 219 insertions(+), 246 deletions(-) create mode 100644 wa/framework/target/assistant.py diff --git a/wa/framework/execution.py b/wa/framework/execution.py index 12b7be8a..ebdf0145 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -138,16 +138,21 @@ class ExecutionContext(object): self.current_job = self.job_queue.pop(0) self.current_job.output = init_job_output(self.run_output, self.current_job) self.update_job_state(self.current_job) + self.tm.start() return self.current_job def end_job(self): if not self.current_job: raise RuntimeError('No jobs in progress') + self.tm.stop() self.completed_jobs.append(self.current_job) self.update_job_state(self.current_job) self.output.write_result() self.current_job = None + def extract_results(self): + self.tm.extract_results(self) + def move_failed(self, job): self.run_output.move_failed(job.output) diff --git a/wa/framework/instrumentation.py b/wa/framework/instrumentation.py index f96c0cec..3fe67de3 100644 --- a/wa/framework/instrumentation.py +++ b/wa/framework/instrumentation.py @@ -125,8 +125,8 @@ SIGNAL_MAP = OrderedDict([ ('setup', signal.BEFORE_WORKLOAD_SETUP), ('start', signal.BEFORE_WORKLOAD_EXECUTION), ('stop', signal.AFTER_WORKLOAD_EXECUTION), - ('process_workload_result', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE), - ('update_result', signal.AFTER_WORKLOAD_RESULT_UPDATE), + ('process_workload_output', signal.SUCCESSFUL_WORKLOAD_OUTPUT_UPDATE), + ('update_result', signal.AFTER_WORKLOAD_OUTPUT_UPDATE), ('teardown', signal.AFTER_WORKLOAD_TEARDOWN), ('finalize', signal.RUN_FINALIZED), diff --git a/wa/framework/job.py b/wa/framework/job.py index 9b7bc7da..880cfff4 100644 --- a/wa/framework/job.py +++ b/wa/framework/job.py @@ -68,8 +68,11 @@ class Job(object): def process_output(self, context): self.logger.info('Processing output for job {}'.format(self.id)) - with signal.wrap('WORKLOAD_RESULT_UPDATE', self, context): - self.workload.update_result(context) + with signal.wrap('WORKLOAD_RESULT_EXTRACTION', self, context): + self.workload.extract_results(context) + context.extract_results() + with signal.wrap('WORKLOAD_OUTPUT_UPDATE', self, context): + self.workload.update_output(context) def teardown(self, context): self.logger.info('Tearing down job {}'.format(self.id)) diff --git a/wa/framework/signal.py b/wa/framework/signal.py index 09fe7b4e..20c6a0b2 100644 --- a/wa/framework/signal.py +++ b/wa/framework/signal.py @@ -97,7 +97,8 @@ JOB_FINALIZED = Signal('job-finalized') # Signals associated with particular stages of workload execution -BEFORE_WORKLOAD_INITIALIZED = Signal('before-workload-initialized', invert_priority=True) +BEFORE_WORKLOAD_INITIALIZED = Signal('before-workload-initialized', + invert_priority=True) SUCCESSFUL_WORKLOAD_INITIALIZED = Signal('successful-workload-initialized') AFTER_WORKLOAD_INITIALIZED = Signal('after-workload-initialized') @@ -109,9 +110,15 @@ BEFORE_WORKLOAD_EXECUTION = Signal('before-workload-execution', invert_priority= SUCCESSFUL_WORKLOAD_EXECUTION = Signal('successful-workload-execution') AFTER_WORKLOAD_EXECUTION = Signal('after-workload-execution') -BEFORE_WORKLOAD_RESULT_UPDATE = Signal('before-workload-result-update', invert_priority=True) -SUCCESSFUL_WORKLOAD_RESULT_UPDATE = Signal('successful-workload-result-update') -AFTER_WORKLOAD_RESULT_UPDATE = Signal('after-workload-result-update') +BEFORE_WORKLOAD_RESULT_EXTRACTION = Signal('before-workload-result-exptracton', + invert_priority=True) +SUCCESSFUL_WORKLOAD_RESULT_EXTRACTION = Signal('successful-workload-result-exptracton') +AFTER_WORKLOAD_RESULT_EXTRACTION = Signal('after-workload-result-exptracton') + +BEFORE_WORKLOAD_OUTPUT_UPDATE = Signal('before-workload-output-update', + invert_priority=True) +SUCCESSFUL_WORKLOAD_OUTPUT_UPDATE = Signal('successful-workload-output-update') +AFTER_WORKLOAD_OUTPUT_UPDATE = Signal('after-workload-output-update') BEFORE_WORKLOAD_TEARDOWN = Signal('before-workload-teardown', invert_priority=True) SUCCESSFUL_WORKLOAD_TEARDOWN = Signal('successful-workload-teardown') diff --git a/wa/framework/target/assistant.py b/wa/framework/target/assistant.py new file mode 100644 index 00000000..99411059 --- /dev/null +++ b/wa/framework/target/assistant.py @@ -0,0 +1,144 @@ +import logging +import os +import shutil +import sys +import tempfile +import threading +import time + +from wa import Parameter +from wa.framework.exception import WorkerThreadError + + +class LinuxAssistant(object): + + parameters = [] + + def __init__(self, target): + self.target = target + + def start(self): + pass + + def extract_results(self, context): + pass + + def stop(self): + pass + + +class AndroidAssistant(object): + + parameters = [ + Parameter('logcat_poll_period', kind=int, + constraint=lambda x: x > 0, + description=""" + Polling period for logcat in seconds. If not specified, + no polling will be used. + + Logcat buffer on android is of limited size and it cannot be + adjusted at run time. Depending on the amount of logging activity, + the buffer may not be enought to capture comlete trace for a + workload execution. For those situations, logcat may be polled + periodically during the course of the run and stored in a + temporary locaiton on the host. Setting the value of the poll + period enables this behavior. + """), + ] + + def __init__(self, target, logcat_poll_period=None): + self.target = target + if logcat_poll_period: + self.logcat_poller = LogcatPoller(target, logcat_poll_period) + else: + self.logcat_poller = None + + def start(self): + if self.logcat_poller: + self.logcat_poller.start() + + def stop(self): + if self.logcat_poller: + self.logcat_poller.stop() + + def extract_results(self, context): + logcat_file = os.path.join(context.output_directory, 'logcat.log') + self.dump_logcat(logcat_file) + context.add_artifact('logcat', logcat_file, kind='log') + self.clear_logcat() + + def dump_logcat(self, outfile): + if self.logcat_poller: + self.logcat_poller.write_log(outfile) + else: + self.target.dump_logcat(outfile) + + def clear_logcat(self): + if self.logcat_poller: + self.logcat_poller.clear_buffer() + + +class LogcatPoller(threading.Thread): + + def __init__(self, target, period=60, timeout=30): + super(LogcatPoller, self).__init__() + self.target = target + self.logger = logging.getLogger('logcat') + self.period = period + self.timeout = timeout + self.stop_signal = threading.Event() + self.lock = threading.Lock() + self.buffer_file = tempfile.mktemp() + self.last_poll = 0 + self.daemon = True + self.exc = None + + def start(self): + self.logger.debug('starting polling') + try: + while True: + if self.stop_signal.is_set(): + break + with self.lock: + current_time = time.time() + if (current_time - self.last_poll) >= self.period: + self.poll() + time.sleep(0.5) + except Exception: # pylint: disable=W0703 + self.exc = WorkerThreadError(self.name, sys.exc_info()) + self.logger.debug('polling stopped') + + def stop(self): + self.logger.debug('Stopping logcat polling') + self.stop_signal.set() + self.join(self.timeout) + if self.is_alive(): + self.logger.error('Could not join logcat poller thread.') + if self.exc: + raise self.exc # pylint: disable=E0702 + + def clear_buffer(self): + self.logger.debug('clearing logcat buffer') + with self.lock: + self.target.clear_logcat() + with open(self.buffer_file, 'w') as _: # NOQA + pass + + def write_log(self, outfile): + with self.lock: + self.poll() + if os.path.isfile(self.buffer_file): + shutil.copy(self.buffer_file, outfile) + else: # there was no logcat trace at this time + with open(outfile, 'w') as _: # NOQA + pass + + def close(self): + self.logger.debug('closing poller') + if os.path.isfile(self.buffer_file): + os.remove(self.buffer_file) + + def poll(self): + self.last_poll = time.time() + self.target.dump_logcat(self.buffer_file, append=True, timeout=self.timeout) + self.target.clear_logcat() diff --git a/wa/framework/target/descriptor.py b/wa/framework/target/descriptor.py index 717df67d..9e01ebac 100644 --- a/wa/framework/target/descriptor.py +++ b/wa/framework/target/descriptor.py @@ -7,6 +7,7 @@ from devlib import (LinuxTarget, AndroidTarget, LocalLinuxTarget, from wa.framework import pluginloader from wa.framework.exception import PluginLoaderError from wa.framework.plugin import Plugin, Parameter +from wa.framework.target.assistant import LinuxAssistant, AndroidAssistant from wa.utils.types import list_of_strings, list_of_ints from wa.utils.misc import isiterable @@ -28,6 +29,7 @@ def instantiate_target(tdesc, params, connect=None): target_params = {p.name: p for p in tdesc.target_params} platform_params = {p.name: p for p in tdesc.platform_params} conn_params = {p.name: p for p in tdesc.conn_params} + assistant_params = {p.name: p for p in tdesc.assistant_params} tp, pp, cp = {}, {}, {} @@ -38,6 +40,8 @@ def instantiate_target(tdesc, params, connect=None): pp[name] = value elif name in conn_params: cp[name] = value + elif name in assistant_params: + pass else: msg = 'Unexpected parameter for {}: {}' raise ValueError(msg.format(tdesc.name, name)) @@ -53,17 +57,27 @@ def instantiate_target(tdesc, params, connect=None): return tdesc.target(**tp) +def instantiate_assistant(tdesc, params, target): + assistant_params = {} + for param in tdesc.assistant_params: + if param.name in params: + assistant_params[param.name] = params[param.name] + return tdesc.assistant(target, **assistant_params) + + class TargetDescription(object): def __init__(self, name, source, description=None, target=None, platform=None, - conn=None, target_params=None, platform_params=None, - conn_params=None): + conn=None, assistant=None, target_params=None, platform_params=None, + conn_params=None, assistant_params=None): self.name = name self.source = source self.description = description self.target = target self.platform = platform self.connection = conn + self.assistant = assistant + self.assistant_params = assistant_params self._set('target_params', target_params) self._set('platform_params', platform_params) self._set('conn_params', conn_params) @@ -218,7 +232,7 @@ GEM5_PLATFORM_PARAMS = [ '''), ] -# name --> (target_class, params_list, defaults) +# name --> (target_class, params_list, defaults, assistant_class) TARGETS = { 'linux': (LinuxTarget, COMMON_TARGET_PARAMS, None), 'android': (AndroidTarget, COMMON_TARGET_PARAMS + @@ -230,6 +244,13 @@ TARGETS = { 'local': (LocalLinuxTarget, COMMON_TARGET_PARAMS, None), } +# name --> assistant +ASSISTANTS = { + 'linux': LinuxAssistant, + 'android': AndroidAssistant, + 'local': LinuxAssistant, +} + # name --> (platform_class, params_list, defaults) PLATFORMS = { 'generic': (Platform, COMMON_PLATFORM_PARAMS, None), @@ -267,6 +288,7 @@ class DefaultTargetDescriptor(TargetDescriptor): result = [] for target_name, target_tuple in TARGETS.iteritems(): target, target_params = self._get_item(target_tuple) + assistant = ASSISTANTS[target_name] for platform_name, platform_tuple in PLATFORMS.iteritems(): platform, platform_params = self._get_item(platform_tuple) @@ -274,8 +296,10 @@ class DefaultTargetDescriptor(TargetDescriptor): td = TargetDescription(name, self) td.target = target td.platform = platform + td.assistant = assistant td.target_params = target_params td.platform_params = platform_params + td.assistant_params = assistant.parameters result.append(td) return result diff --git a/wa/framework/target/manager.py b/wa/framework/target/manager.py index 043dbcb4..f1f233c9 100644 --- a/wa/framework/target/manager.py +++ b/wa/framework/target/manager.py @@ -10,7 +10,8 @@ from wa.framework import signal from wa.framework.exception import WorkerThreadError, ConfigError from wa.framework.plugin import Parameter from wa.framework.target.descriptor import (get_target_descriptions, - instantiate_target) + instantiate_target, + instantiate_assistant) from wa.framework.target.info import TargetInfo from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig, HotplugRuntimeConfig, @@ -62,7 +63,6 @@ class TargetManager(object): self.info = TargetInfo() self._init_target() - self._init_assistant() self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] def finalize(self): @@ -83,6 +83,15 @@ class TargetManager(object): if any(parameter in name for parameter in cfg.supported_parameters): cfg.add(name, self.parameters.pop(name)) + def start(self): + self.assistant.start() + + def stop(self): + self.assistant.stop() + + def extract_results(self, context): + self.assistant.extract_results(context) + @memoized def get_target_info(self): return TargetInfo(self.target) @@ -107,238 +116,12 @@ class TargetManager(object): if self.target_name not in target_map: raise ValueError('Unknown Target: {}'.format(self.target_name)) tdesc = target_map[self.target_name] + self.target = instantiate_target(tdesc, self.parameters, connect=False) + with signal.wrap('TARGET_CONNECT'): self.target.connect() self.logger.info('Setting up target') self.target.setup() - def _init_assistant(self): - # Create a corresponding target and target-assistant to help with - # platformy stuff? - if self.target.os == 'android': - self.assistant = AndroidAssistant(self.target) - elif self.target.os == 'linux': - self.assistant = LinuxAssistant(self.target) # pylint: disable=redefined-variable-type - else: - raise ValueError('Unknown Target OS: {}'.format(self.target.os)) - - -class LinuxAssistant(object): - - name = 'linux-assistant' - - description = """ - Performs configuration, instrumentation, etc. during runs on Linux targets. - """ - - def __init__(self, target, **kwargs): - self.target = target - # parameters = [ - - # Parameter('disconnect', kind=bool, default=False, - # description=""" - # Specifies whether the target should be disconnected from - # at the end of the run. - # """), - # ] - - # runtime_config_cls = [ - # # order matters - # SysfileValuesRuntimeConfig, - # HotplugRuntimeConfig, - # CpufreqRuntimeConfig, - # CpuidleRuntimeConfig, - # ] - - # def __init__(self, target, context, **kwargs): - # # super(LinuxTargetManager, self).__init__(target, context, **kwargs) - # self.target = target - # self.context = context - # self.info = TargetInfo() - # self.runtime_configs = [cls(target) for cls in self.runtime_config_cls] - - # def __init__(self): - # # super(LinuxTargetManager, self).__init__(target, context, **kwargs) - # self.target = target - # self.info = TargetInfo() - # self.parameters = parameters - - # self.info = TargetInfo() - # self.runtime_configs = [cls(target) for cls in self.runtime_config_cls] - - # def initialize(self): - # # self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] - # # if self.parameters: - # self.logger.info('Connecting to the device') - # with signal.wrap('TARGET_CONNECT'): - # self.target.connect() - # self.info.load(self.target) - # # info_file = os.path.join(self.context.info_directory, 'target.json') - # # with open(info_file, 'w') as wfh: - # # json.dump(self.info.to_pod(), wfh) - - # def finalize(self, runner): - # self.logger.info('Disconnecting from the device') - # if self.disconnect: - # with signal.wrap('TARGET_DISCONNECT'): - # self.target.disconnect() - - # def _add_parameters(self): - # for name, value in self.parameters.iteritems(): - # self.add_parameter(name, value) - - # def validate_runtime_parameters(self, parameters): - # self.clear() - # for name, value in parameters.iteritems(): - # self.add_parameter(name, value) - # self.validate_parameters() - - # def set_runtime_parameters(self, parameters): - # self.clear() - # for name, value in parameters.iteritems(): - # self.add_parameter(name, value) - # self.set_parameters() - - # def clear_parameters(self): - # for cfg in self.runtime_configs: - # cfg.clear() - - # def add_parameter(self, name, value): - # for cfg in self.runtime_configs: - # if name in cfg.supported_parameters: - # cfg.add(name, value) - # return - # raise ConfigError('Unexpected runtime parameter "{}".'.format(name)) - - # def validate_parameters(self): - # for cfg in self.runtime_configs: - # cfg.validate() - - # def set_parameters(self): - # for cfg in self.runtime_configs: - # cfg.set() - - -class AndroidAssistant(LinuxAssistant): - - name = 'android-assistant' - description = """ - Extends ``LinuxTargetManager`` with Android-specific operations. - """ - - parameters = [ - Parameter('logcat_poll_period', kind=int, - description=""" - If specified, logcat will cached in a temporary file on the - host every ``logcat_poll_period`` seconds. This is useful for - longer job executions, where on-device logcat buffer may not be - big enough to capture output for the entire execution. - """), - ] - - def __init__(self, target, **kwargs): - super(AndroidAssistant, self).__init__(target) - self.logcat_poll_period = kwargs.get('logcat_poll_period', None) - if self.logcat_poll_period: - self.logcat_poller = LogcatPoller(target, self.logcat_poll_period) - else: - self.logcat_poller = None - - # def __init__(self, target, context, **kwargs): - # super(AndroidAssistant, self).__init__(target, context, **kwargs) - # self.logcat_poll_period = kwargs.get('logcat_poll_period', None) - # if self.logcat_poll_period: - # self.logcat_poller = LogcatPoller(target, self.logcat_poll_period) - # else: - # self.logcat_poller = None - - # def next_job(self, job): - # super(AndroidAssistant, self).next_job(job) - # if self.logcat_poller: - # self.logcat_poller.start() - - # def job_done(self, job): - # super(AndroidAssistant, self).job_done(job) - # if self.logcat_poller: - # self.logcat_poller.stop() - # outfile = os.path.join(self.context.output_directory, 'logcat.log') - # self.logger.debug('Dumping logcat to {}'.format(outfile)) - # self.dump_logcat(outfile) - # self.clear() - - def dump_logcat(self, outfile): - if self.logcat_poller: - self.logcat_poller.write_log(outfile) - else: - self.target.dump_logcat(outfile) - - def clear_logcat(self): - if self.logcat_poller: - self.logcat_poller.clear_buffer() - - -class LogcatPoller(threading.Thread): - - def __init__(self, target, period=60, timeout=30): - super(LogcatPoller, self).__init__() - self.target = target - self.logger = logging.getLogger('logcat') - self.period = period - self.timeout = timeout - self.stop_signal = threading.Event() - self.lock = threading.Lock() - self.buffer_file = tempfile.mktemp() - self.last_poll = 0 - self.daemon = True - self.exc = None - - def start(self): - self.logger.debug('starting polling') - try: - while True: - if self.stop_signal.is_set(): - break - with self.lock: - current_time = time.time() - if (current_time - self.last_poll) >= self.period: - self.poll() - time.sleep(0.5) - except Exception: # pylint: disable=W0703 - self.exc = WorkerThreadError(self.name, sys.exc_info()) - self.logger.debug('polling stopped') - - def stop(self): - self.logger.debug('Stopping logcat polling') - self.stop_signal.set() - self.join(self.timeout) - if self.is_alive(): - self.logger.error('Could not join logcat poller thread.') - if self.exc: - raise self.exc # pylint: disable=E0702 - - def clear_buffer(self): - self.logger.debug('clearing logcat buffer') - with self.lock: - self.target.clear_logcat() - with open(self.buffer_file, 'w') as _: # NOQA - pass - - def write_log(self, outfile): - with self.lock: - self.poll() - if os.path.isfile(self.buffer_file): - shutil.copy(self.buffer_file, outfile) - else: # there was no logcat trace at this time - with open(outfile, 'w') as _: # NOQA - pass - - def close(self): - self.logger.debug('closing poller') - if os.path.isfile(self.buffer_file): - os.remove(self.buffer_file) - - def poll(self): - self.last_poll = time.time() - self.target.dump_logcat(self.buffer_file, append=True, timeout=self.timeout) - self.target.clear_logcat() + self.assistant = instantiate_assistant(tdesc, self.parameters, self.target) diff --git a/wa/framework/workload.py b/wa/framework/workload.py index 850bceca..842e9caa 100644 --- a/wa/framework/workload.py +++ b/wa/framework/workload.py @@ -66,10 +66,16 @@ class Workload(TargetedPlugin): """ pass - def update_result(self, context): + def extract_results(self, context): """ - Update the result within the specified execution context with the - metrics form this workload iteration. + Extract results on the target + """ + pass + + def update_output(self, context): + """ + Update the output within the specified execution context with the + metrics and artifacts form this workload iteration. """ pass diff --git a/wa/workloads/dhrystone/__init__.py b/wa/workloads/dhrystone/__init__.py index 106d0483..369e84bf 100644 --- a/wa/workloads/dhrystone/__init__.py +++ b/wa/workloads/dhrystone/__init__.py @@ -105,12 +105,13 @@ class Dhrystone(Workload): self.target.killall('dhrystone') raise - def update_result(self, context): + def extract_results(self, context): outfile = os.path.join(context.output_directory, 'dhrystone.output') with open(outfile, 'w') as wfh: wfh.write(self.output) context.add_artifact('dhrystone-output', outfile, 'raw', "dhrystone's stdout") + def update_output(self, context): score_count = 0 dmips_count = 0 total_score = 0 From 4006e998c2a83c6b942dcf101b66a5638e42af45 Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Tue, 28 Mar 2017 09:41:42 +0100 Subject: [PATCH 2/5] output: fix JobOutput instantiation iteration and label parameters were being passed in the wrong order when instantiating JobOutput. --- wa/framework/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wa/framework/output.py b/wa/framework/output.py index daf3d83a..5f72ccb5 100644 --- a/wa/framework/output.py +++ b/wa/framework/output.py @@ -465,7 +465,7 @@ def init_job_output(run_output, job): path = os.path.join(run_output.basepath, output_name) ensure_directory_exists(path) write_pod(Result().to_pod(), os.path.join(path, 'result.json')) - job_output = JobOutput(path, job.id, job.iteration, job.label, job.retries) + job_output = JobOutput(path, job.id, job.label, job.iteration, job.retries) job_output.status = job.status run_output.jobs.append(job_output) return job_output From fed454fc7401cf54b0fb6d7c9c69d92f62897376 Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Tue, 28 Mar 2017 09:44:24 +0100 Subject: [PATCH 3/5] getters: fix some issues - get_by_extension was comparing the expected extension to the entire tuple returned by os.path.splitext(), rather than just the extension part. - UserDirectory getter was looking in the root dependencies directory, rather than the subdirectory for the owner. - Filer getter was not handling non-existing paths properly. --- wa/framework/getters.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wa/framework/getters.py b/wa/framework/getters.py index 9ce5f94e..651ca5a2 100644 --- a/wa/framework/getters.py +++ b/wa/framework/getters.py @@ -52,7 +52,7 @@ def get_by_extension(path, ext): found = [] for entry in os.listdir(path): - entry_ext = os.path.splitext(entry) + entry_ext = os.path.splitext(entry)[1] if entry_ext == ext: found.append(os.path.join(path, entry)) return found @@ -112,7 +112,8 @@ class UserDirectory(ResourceGetter): def get(self, resource): basepath = settings.dependencies_directory - return get_from_location(basepath, resource) + directory = _d(os.path.join(basepath, resource.owner.name)) + return get_from_location(directory, resource) class Http(ResourceGetter): @@ -318,7 +319,9 @@ class Filer(ResourceGetter): result = get_from_location(local_path, resource) if result: return result - if remote_path: + if not os.path.exists(local_path): + return None + if os.path.exists(remote_path): # Didn't find it cached locally; now check the remoted result = get_from_location(remote_path, resource) if not result: From 6fba05503d9081bcd04191f644ea78dd5d951081 Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Tue, 28 Mar 2017 09:53:35 +0100 Subject: [PATCH 4/5] gitignore: add uiautomator generated files --- .gitignore | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 1296a7fe..e41d4738 100755 --- a/.gitignore +++ b/.gitignore @@ -14,17 +14,10 @@ wa_output/ doc/source/api/ doc/source/plugins/ MANIFEST -wlauto/external/uiautomator/bin/ -wlauto/external/uiautomator/*.properties -wlauto/external/uiautomator/build.xml *.orig local.properties -wlauto/external/revent/libs/ -wlauto/external/revent/obj/ -wlauto/external/bbench_server/libs/ -wlauto/external/bbench_server/obj/ pmu_logger.mod.c .tmp_versions obj/ libs/armeabi -wlauto/workloads/*/uiauto/bin/ +uiauto/bin/ From 18e7ffb8268fa01e78ad0ab2daf88462f5159c1a Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Tue, 28 Mar 2017 09:58:48 +0100 Subject: [PATCH 5/5] workload: adding basic UIAutomator workload implementation Added a workload type to handle workloads that have both an APK with an application and associated automation JAR. Added benchmarkpi implementation using using the new workload. --- wa/__init__.py | 2 +- wa/framework/execution.py | 18 +- wa/framework/output.py | 13 ++ wa/framework/resource.py | 2 +- wa/framework/uiauto/BaseUiAutomation.class | Bin 0 -> 4500 bytes wa/framework/uiauto/build.sh | 21 ++ wa/framework/uiauto/build.xml | 92 +++++++++ wa/framework/uiauto/project.properties | 14 ++ .../com/arm/wa/uiauto/BaseUiAutomation.java | 124 ++++++++++++ wa/framework/workload.py | 191 ++++++++++++------ wa/workloads/benchmarkpi/__init__.py | 62 ++++++ .../com.arm.wa.uiauto.benchmarkpi.jar | Bin 0 -> 3071 bytes wa/workloads/benchmarkpi/uiauto/build.sh | 27 +++ wa/workloads/benchmarkpi/uiauto/build.xml | 92 +++++++++ .../benchmarkpi/uiauto/project.properties | 14 ++ .../src/com/arm/wa/uiauto/UiAutomation.java | 62 ++++++ 16 files changed, 670 insertions(+), 64 deletions(-) create mode 100644 wa/framework/uiauto/BaseUiAutomation.class create mode 100755 wa/framework/uiauto/build.sh create mode 100644 wa/framework/uiauto/build.xml create mode 100644 wa/framework/uiauto/project.properties create mode 100644 wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java create mode 100644 wa/workloads/benchmarkpi/__init__.py create mode 100644 wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar create mode 100755 wa/workloads/benchmarkpi/uiauto/build.sh create mode 100644 wa/workloads/benchmarkpi/uiauto/build.xml create mode 100644 wa/workloads/benchmarkpi/uiauto/project.properties create mode 100644 wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java diff --git a/wa/__init__.py b/wa/__init__.py index cf6a4d2e..d005ef27 100644 --- a/wa/__init__.py +++ b/wa/__init__.py @@ -14,4 +14,4 @@ from wa.framework.plugin import Plugin, Parameter from wa.framework.processor import ResultProcessor from wa.framework.resource import (NO_ONE, JarFile, ApkFile, ReventFile, File, Executable) -from wa.framework.workload import Workload +from wa.framework.workload import Workload, ApkUiautoWorkload diff --git a/wa/framework/execution.py b/wa/framework/execution.py index ebdf0145..d9fc5132 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -30,7 +30,7 @@ import wa.framework.signal as signal from wa.framework import instrumentation, pluginloader from wa.framework.configuration.core import settings, Status from wa.framework.exception import (WAError, ConfigError, TimeoutError, - InstrumentError, TargetError, + InstrumentError, TargetError, HostError, TargetNotRespondingError) from wa.framework.output import init_job_output from wa.framework.plugin import Artifact @@ -178,6 +178,22 @@ class ExecutionContext(object): classifiers) self.output.add_metric(name, value, units, lower_is_better, classifiers) + def get_artifact(self, name): + try: + return self.output.get_artifact(name) + except HostError: + if not self.current_job: + raise + return self.run_output.get_artifact(name) + + def get_artifact_path(self, name): + try: + return self.output.get_artifact_path(name) + except HostError: + if not self.current_job: + raise + return self.run_output.get_artifact_path(name) + def add_artifact(self, name, path, kind, description=None, classifiers=None): self.output.add_artifact(name, path, kind, description, classifiers) diff --git a/wa/framework/output.py b/wa/framework/output.py index 5f72ccb5..53c9fd3d 100644 --- a/wa/framework/output.py +++ b/wa/framework/output.py @@ -84,6 +84,13 @@ class Output(object): def add_event(self, message): self.result.add_event(message) + def get_artifact(self, name): + return self.result.get_artifact(name) + + def get_artifact_path(self, name): + artifact = self.get_artifact(name) + return self.get_path(artifact.path) + class RunOutput(Output): @@ -234,6 +241,12 @@ class Result(object): def add_event(self, message): self.events.append(Event(message)) + def get_artifact(self, name): + for artifact in self.artifacts: + if artifact.name == name: + return artifact + raise HostError('Artifact "{}" not found'.format(name)) + def to_pod(self): return dict( status=str(self.status), diff --git a/wa/framework/resource.py b/wa/framework/resource.py index 2ce9de7f..44745386 100644 --- a/wa/framework/resource.py +++ b/wa/framework/resource.py @@ -75,7 +75,7 @@ class Resource(object): raise NotImplementedError() def __str__(self): - return '<{}\'s {}>'.format(self.owner, self.name) + return '<{}\'s {}>'.format(self.owner, self.kind) class File(Resource): diff --git a/wa/framework/uiauto/BaseUiAutomation.class b/wa/framework/uiauto/BaseUiAutomation.class new file mode 100644 index 0000000000000000000000000000000000000000..942f7ca3da0c5f040669e12e90cf612a311178fc GIT binary patch literal 4500 zcmb7H`(qSW75*l>$?VP!&@3eo($;NAAn%n&(-PWR2&9IFfC+&HYUwb0lT6smq_eXm z)VCFj^{w?mAHEeIsIAsbY;5bZDn9=VfAc@Ees^YecOb#VA9iN$x#xW6eeT_B|9R~V z06Xzd4SQhAhofOP<`Z~QgNuS(cq)o2&TB|WjszAGSd{Mz8dN-`;eB|&iVsN6532Z( zhDO*bJ}loKN#LXM{j`RS_?YClD8D|g;+X_4$Bu3>s(qEO6xL{E6aaC6RrEfMlU5Fy?%ERAg_2-C4CqYP8dGtP-| z(-p!l%s8HcO&ue%#(5)~H|*)`xaV5-^uEqhWTrQFMz%(s@uE3%M0hh!?vU#`E=`Tw z#+>i;Dg?}1< zNR#FUD+OI0wQi3S)_N9Vba<)OAcz@H1YmhhqVefopJu zkrPbOmY}LER)2A7O1L67=I@XvWfomm*db#5l*4kNCc2@GJ4M$NLze8aO*ghppH!&h z9`x%tfbZ(~9aYz`Z$Z^6n< z3$IUR=7c2{bo>IZN-Ik`PU4h~B`hoGza{8V#~YH-t_&TjnXm%28LN=79WUdjV<4mB zmv~L8Ii_O;%PM}Q<8{2jYRLOlmgzNhyoq0{_>GR=;&)7au%V!-9Qx}sub{cSF>KEl zJ?23eb3s+8_`Qxl;4P`}kMi*+nc_hmf5yuyuIcy-{;K0|=w}dpkRqFf&2ERKl$F)V zf{i$~U6OJc5;L;BIPvJ3n+bX(BZoq!wY|`);I?Xt%L({9{-L0?_8|WsarZ>Q<1Dv@ z%W)Kre#bpd>L5=84$P|7h|%#5zbBJS__1)ZZtCb772uicp$)e?I4VI!#tj+vai!(G?v<@!r-hC&!g<%v=e=BI`BwO@?^=TL3P0*_AD<0A zu@!sytp^Mkz&@@N+)v>;U%1;}lXjzEdqhwP1s@=DoZ=7i_paIy9W@{xstwUy17d$| zh`t&SJTvHy4CEkUj0%wIYH5u9Yv$q->Z(!E{H>kPu*(N*3RuI)PzVQch(HQ20S||O zFEI)A1l-+KLhMLSI$lEkD8Cz4picI#AThawWOoT#I>ko``V!Wau>R^mJbslBI{#>7 zv4M(YKAU`~d-&|<&R*^e@DJ8L-)!kV4XbHF7==nqZXSP4$d9tD+$;*dSQLc`yKJZuwe31sp7;IZX zBi&7pu3+P2FP*+Uy{UwzC2SsucQwcFC}B%gW#H>UsyxJhZ$oHdNL%ThtYzsd;VSC4 zh3X|url5{eEf~jfdep-237nt_QR7bH(VKW3F(JtJBLo<^iPw_^zTRuztwP`nR2ie0 z+3q|1QYxYOd9tvYqbPF@iDA4aa%elYn&1wYFejblv9W!(nJ?x`L()&yRTk$@N zzzjJC0`H@qp2lM|QRa7;a?+qw#h`awDhg5+1wn<^5%9g0uDg#2a`tG; z$fWNsq4i|dg&=~bki=;pIT%7(h+H;P&XUJ*CWxCd#N_J4yR?jL_r@h?8$k!yFzt-o z_Q~dk5_T-1V+oxDs^m$=`cqBGrntyF_ab(6)tAuKtd`I{Igoe@>E;AO+;jQuf5-b% zR}kBjx{QV^Ncn~!${Mv$7$agw&W9N zgNaV$&;!9}OtBGXSg00TY8GQS$Fg%+4fBkZOO7I`3-GXrXYiCCjmH?5K^FBB?45-4 zxDxS8F#LX!w6oMwN1abolgX0UfD1&M;#_GXWtuuwT5*PYP2%-1-Lur4s+7BoZl36M GnENl(S1#lL literal 0 HcmV?d00001 diff --git a/wa/framework/uiauto/build.sh b/wa/framework/uiauto/build.sh new file mode 100755 index 00000000..1beddd6a --- /dev/null +++ b/wa/framework/uiauto/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + + +ant build + +cp bin/classes/com/arm/wa/uiauto/BaseUiAutomation.class . diff --git a/wa/framework/uiauto/build.xml b/wa/framework/uiauto/build.xml new file mode 100644 index 00000000..86b8aa0f --- /dev/null +++ b/wa/framework/uiauto/build.xml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/wa/framework/uiauto/project.properties b/wa/framework/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wa/framework/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java b/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java new file mode 100644 index 00000000..fc0faded --- /dev/null +++ b/wa/framework/uiauto/src/com/arm/wa/uiauto/BaseUiAutomation.java @@ -0,0 +1,124 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wa.uiauto; + +import java.io.File; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.concurrent.TimeoutException; + +import android.app.Activity; +import android.os.Bundle; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +public class BaseUiAutomation extends UiAutomatorTestCase { + + public void setup() throws Exception { + } + + public void runWorkload() throws Exception { + } + + public void extractResults() throws Exception { + } + + public void teardown() throws Exception { + } + + public void sleep(int second) { + super.sleep(second * 1000); + } + + public boolean takeScreenshot(String name) { + Bundle params = getParams(); + String png_dir = params.getString("workdir"); + + try { + return getUiDevice().takeScreenshot(new File(png_dir, name + ".png")); + } catch(NoSuchMethodError e) { + return true; + } + } + + public void waitText(String text) throws UiObjectNotFoundException { + waitText(text, 600); + } + + public void waitText(String text, int second) throws UiObjectNotFoundException { + UiSelector selector = new UiSelector(); + UiObject text_obj = new UiObject(selector.text(text) + .className("android.widget.TextView")); + waitObject(text_obj, second); + } + + public void waitObject(UiObject obj) throws UiObjectNotFoundException { + waitObject(obj, 600); + } + + public void waitObject(UiObject obj, int second) throws UiObjectNotFoundException { + if (! obj.waitForExists(second * 1000)){ + throw new UiObjectNotFoundException("UiObject is not found: " + + obj.getSelector().toString()); + } + } + + public boolean waitUntilNoObject(UiObject obj, int second) { + return obj.waitUntilGone(second * 1000); + } + + public void clearLogcat() throws Exception { + Runtime.getRuntime().exec("logcat -c"); + } + + public void waitForLogcatText(String searchText, long timeout) throws Exception { + long startTime = System.currentTimeMillis(); + Process process = Runtime.getRuntime().exec("logcat"); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + + long currentTime = System.currentTimeMillis(); + boolean found = false; + while ((currentTime - startTime) < timeout){ + sleep(2); // poll every two seconds + + while((line=reader.readLine())!=null) { + if (line.contains(searchText)) { + found = true; + break; + } + } + + if (found) { + break; + } + currentTime = System.currentTimeMillis(); + } + + process.destroy(); + + if ((currentTime - startTime) >= timeout) { + throw new TimeoutException("Timed out waiting for Logcat text \"%s\"".format(searchText)); + } + } +} + diff --git a/wa/framework/workload.py b/wa/framework/workload.py index 842e9caa..b2f565e3 100644 --- a/wa/framework/workload.py +++ b/wa/framework/workload.py @@ -13,9 +13,11 @@ # limitations under the License. # import logging +import os +import time from wa.framework.plugin import TargetedPlugin -from wa.framework.resource import JarFile, ReventFile, NO_ONE +from wa.framework.resource import ApkFile, JarFile, ReventFile, NO_ONE from wa.framework.exception import WorkloadError from devlib.utils.android import ApkInfo @@ -91,53 +93,117 @@ class Workload(TargetedPlugin): return ''.format(self.name) -class UiAutomatorGUI(object): +class ApkUiautoWorkload(Workload): + + platform = 'android' - def __init__(self, target, package='', klass='UiAutomation', - method='runUiAutoamtion'): - self.target = target - self.uiauto_package = package - self.uiauto_class = klass - self.uiauto_method = method - self.uiauto_file = None - self.target_uiauto_file = None - self.command = None - self.uiauto_params = {} + def __init__(self, target, **kwargs): + super(ApkUiautoWorkload, self).__init__(target, **kwargs) + self.apk = ApkHander(self) + self.gui = UiAutomatorGUI(self) def init_resources(self, context): - self.uiauto_file = context.resolver.get(JarFile(self)) - self.target_uiauto_file = self.target.path.join(self.target.working_directory, - os.path.basename(self.uiauto_file)) - if not self.uiauto_package: - self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] + self.apk.init_resources(context.resolver) + self.gui.init_resources(context.resolver) + self.gui.init_commands() - def validate(self): - if not self.uiauto_file: - raise WorkloadError('No UI automation JAR file found for workload {}.'.format(self.name)) - if not self.uiauto_package: - raise WorkloadError('No UI automation package specified for workload {}.'.format(self.name)) + def initialize(self, context): + self.gui.deploy() def setup(self, context): - method_string = '{}.{}#{}'.format(self.uiauto_package, self.uiauto_class, self.uiauto_method) + self.apk.setup(context) + self.gui.setup() + + def run(self, context): + self.gui.run() + + def extract_results(self, context): + self.gui.extract_results() + + def teardown(self, context): + self.gui.teardown() + self.apk.teardown() + + def finalize(self, context): + self.gui.remove() + + +class UiAutomatorGUI(object): + + stages = ['setup', 'runWorkload', 'extractResults', 'teardown'] + + def __init__(self, owner, package=None, klass='UiAutomation', timeout=600): + self.owner = owner + self.target = self.owner.target + self.uiauto_package = package + self.uiauto_class = klass + self.timeout = timeout + self.logger = logging.getLogger('gui') + self.jar_file = None + self.target_jar_file = None + self.commands = {} + self.uiauto_params = {} + + def init_resources(self, resolver): + self.jar_file = resolver.get(JarFile(self.owner)) + jar_name = os.path.basename(self.jar_file) + self.target_jar_file = self.target.get_workpath(jar_name) + if not self.uiauto_package: + package = os.path.splitext(os.path.basename(self.jar_file))[0] + self.uiauto_package = package + + def init_commands(self): params_dict = self.uiauto_params params_dict['workdir'] = self.target.working_directory params = '' for k, v in self.uiauto_params.iteritems(): params += ' -e {} {}'.format(k, v) - self.command = 'uiautomator runtest {}{} -c {}'.format(self.target_uiauto_file, params, method_string) - self.target.push_file(self.uiauto_file, self.target_uiauto_file) - self.target.killall('uiautomator') - def run(self, context): - result = self.target.execute(self.command, self.run_timeout) + for stage in self.stages: + method_string = '{}.{}#{}'.format(self.uiauto_package, + self.uiauto_class, + stage) + cmd_template = 'uiautomator runtest {}{} -c {}' + self.commands[stage] = cmd_template.format(self.target_jar_file, + params, method_string) + + def deploy(self): + self.target.push(self.jar_file, self.target_jar_file) + + def set(self, name, value): + self.uiauto_params[name] = value + + def setup(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self.target.killall('uiautomator') + self._execute('setup', timeout or self.timeout) + + def run(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('runWorkload', timeout or self.timeout) + + def extract_results(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('extractResults', timeout or self.timeout) + + def teardown(self, timeout=None): + if not self.commands: + raise RuntimeError('Commands have not been initialized') + self._execute('teardown', timeout or self.timeout) + + def remove(self): + self.target.remove(self.target_jar_file) + + def _execute(self, stage, timeout): + result = self.target.execute(self.commands[stage], timeout) if 'FAILURE' in result: raise WorkloadError(result) else: self.logger.debug(result) - time.sleep(DELAY) - - def teardown(self, context): - self.target.delete_file(self.target_uiauto_file) + time.sleep(2) class ReventGUI(object): @@ -197,21 +263,27 @@ class ReventGUI(object): class ApkHander(object): - def __init__(self, owner, target, view, install_timeout=300, version=None, + def __init__(self, owner, install_timeout=300, version=None, variant=None, strict=True, force_install=False, uninstall=False): self.logger = logging.getLogger('apk') self.owner = owner - self.target = target + self.target = self.owner.target + self.install_timeout = install_timeout self.version = version + self.variant = variant + self.strict = strict + self.force_install = force_install + self.uninstall = uninstall self.apk_file = None self.apk_info = None self.apk_version = None self.logcat_log = None - def init_resources(self, context): - self.apk_file = context.resolver.get(ApkFile(self.owner), - version=self.version, - strict=strict) + def init_resources(self, resolver): + self.apk_file = resolver.get(ApkFile(self.owner, + variant=self.variant, + version=self.version), + strict=self.strict) self.apk_info = ApkInfo(self.apk_file) def setup(self, context): @@ -226,16 +298,17 @@ class ApkHander(object): self.initialize_with_host_apk(context, installed_version) else: if not installed_version: - message = '''{} not found found on the device and check_apk is set to "False" - so host version was not checked.''' - raise WorkloadError(message.format(self.package)) + message = '{} not found found on the device and check_apk is set '\ + 'to "False" so host version was not checked.' + raise WorkloadError(message.format(self.apk_info.package)) message = 'Version {} installed on device; skipping host APK check.' self.logger.debug(message.format(installed_version)) self.reset(context) self.version = installed_version def initialize_with_host_apk(self, context, installed_version): - if installed_version != self.apk_file.version_name: + host_version = self.apk_info.version_name + if installed_version != host_version: if installed_version: message = '{} host version: {}, device version: {}; re-installing...' self.logger.debug(message.format(os.path.basename(self.apk_file), @@ -251,42 +324,38 @@ class ApkHander(object): host_version)) if self.force_install: if installed_version: - self.device.uninstall(self.package) + self.target.uninstall_package(self.apk_info.package) self.install_apk(context) else: self.reset(context) self.apk_version = host_version def start_activity(self): - output = self.device.execute('am start -W -n {}/{}'.format(self.package, self.activity)) + cmd = 'am start -W -n {}/{}' + output = self.target.execute(cmd.format(self.apk_info.package, + self.apk_info.activity)) if 'Error:' in output: - self.device.execute('am force-stop {}'.format(self.package)) # this will dismiss any erro dialogs + # this will dismiss any error dialogs + self.target.execute('am force-stop {}'.format(self.apk_info.package)) raise WorkloadError(output) self.logger.debug(output) def reset(self, context): # pylint: disable=W0613 - self.device.execute('am force-stop {}'.format(self.package)) - self.device.execute('pm clear {}'.format(self.package)) + self.target.execute('am force-stop {}'.format(self.apk_info.package)) + self.target.execute('pm clear {}'.format(self.apk_info.package)) def install_apk(self, context): - output = self.device.install(self.apk_file, self.install_timeout) + output = self.target.install_apk(self.apk_file, self.install_timeout) if 'Failure' in output: if 'ALREADY_EXISTS' in output: - self.logger.warn('Using already installed APK (did not unistall properly?)') + msg = 'Using already installed APK (did not unistall properly?)' + self.logger.warn(msg) else: raise WorkloadError(output) else: self.logger.debug(output) - def update_result(self, context): - self.logcat_log = os.path.join(context.output_directory, 'logcat.log') - self.device.dump_logcat(self.logcat_log) - context.add_iteration_artifact(name='logcat', - path='logcat.log', - kind='log', - description='Logact dump for the run.') - - def teardown(self, context): - self.device.execute('am force-stop {}'.format(self.package)) - if self.uninstall_apk: - self.device.uninstall(self.package) + def teardown(self): + self.target.execute('am force-stop {}'.format(self.apk_info.package)) + if self.uninstall: + self.target.uninstall_package(self.apk_info.package) diff --git a/wa/workloads/benchmarkpi/__init__.py b/wa/workloads/benchmarkpi/__init__.py new file mode 100644 index 00000000..d731305e --- /dev/null +++ b/wa/workloads/benchmarkpi/__init__.py @@ -0,0 +1,62 @@ +# Copyright 2013-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import re + +from wa import ApkUiautoWorkload + + +class BenchmarkPi(ApkUiautoWorkload): + + name = 'benchmarkpi' + description = """ + Measures the time the target device takes to run and complete the Pi + calculation algorithm. + + http://androidbenchmark.com/howitworks.php + + from the website: + + The whole idea behind this application is to use the same Pi calculation + algorithm on every Android Device and check how fast that proccess is. + Better calculation times, conclude to faster Android devices. This way you + can also check how lightweight your custom made Android build is. Or not. + + As Pi is an irrational number, Benchmark Pi does not calculate the actual Pi + number, but an approximation near the first digits of Pi over the same + calculation circles the algorithms needs. + + So, the number you are getting in miliseconds is the time your mobile device + takes to run and complete the Pi calculation algorithm resulting in a + approximation of the first Pi digits. + """ + package = 'gr.androiddev.BenchmarkPi' + activity = '.BenchmarkPi' + + regex = re.compile('You calculated Pi in ([0-9]+)') + + def update_output(self, context): + super(BenchmarkPi, self).update_output(context) + logcat_file = context.get_artifact_path('logcat') + with open(logcat_file) as fh: + for line in fh: + match = self.regex.search(line) + if match: + result = int(match.group(1)) + + if result is not None: + context.add_metric('pi calculation', result, + 'milliseconds', lower_is_better=True) diff --git a/wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar b/wa/workloads/benchmarkpi/com.arm.wa.uiauto.benchmarkpi.jar new file mode 100644 index 0000000000000000000000000000000000000000..e597b3b515e74da19a45a7bcd31564caa95efa5c GIT binary patch literal 3071 zcmZ{mc{~*A8pj7w)EP}UT1?X%jrCwEj%`xJ*cutjSi;Cwc41HfC$JJ)irz_jy0>^ZfDte((Ex|NH9eavb3S0FD6wr*-^w0DmiP zz!3mS!$?(1TMsF--wFWe|4VudAb2SK0JVYH9?CrqH~-=OONvs}(?)6-8cCy&sK`>_ zbvQp@Lu4-``h;sZ=Q)Q`v#Iegk(EJ&^DA(hf>9@BG2d5(U>52Jme`h$d?vkt(|^PpdSdtAEgS9-S+^J*+-AKKlPEbSW(mH@sI*1{?nOmc{5p3oL4`da zuIqt6(%5B-<&fhAsc7`+p;*mifwwDwmK~KuOvI*xBP1JW0h)l4!oK0fZHSHG(Z^F> zb3?C`VOvgpmK@-B?iZcSi@jtZGUEU%(&>HH-kD~4)Qo=JVMMrBZPZIKchU_iS7WI~ zkZ27w`edv2)p%#>gS#71z%yu8?bjAQQobEzOKn}UI$jSHqgmNf`({PBsEJ;=esFW|cMpoV28lh{-rZpTjWZ+(djDK#I)`B-<)VYao+ zGf?|9u5l*t%TD2~kD3d-H>LpCN`fh!RS<_LYQ0kEFSW@QsrzX1s>~y1{0a`5(#eu& z-D6IMKy3;eTALr`iR~a@B)BHbCkWvzoSAAO6*el_y1%0CI)p8;+0hJ);R3k3)AE?2f;e^b&TK?xVj_Or|yJgd}I!n!Z&NyUFY^PI?eM?`{uIK)pLM@j>s8 zHH}u7t^%fN;ra&8ZT*+Je`uEEFtzDj?<&j!!bVXJ^p?k)yBH4{GdixcPC;tnhi~-Hf!06JyUYa&#&i}162xq?3`F~V&wyH zPg^7;ML1nf3yX{ZEo9ZB=&vg;*J+vAzFIDcL3zExsLkM}W!!wc$ppp9>=gZ>go;4f7@v8#~CQGgln-q(ozv{-07=n_m^6CLg<~jhGBb^Hz!rH z6L{;ziO`03vd8_6pP#|fg}~~Ius!6orNT!~^X@LjOL~TUf@Y81P6(=6MtUyZ3$E2H zlvqX-#a^CTEq?Z(8eILUjpEj{h2@87CmgV3r@Wk#wTY+64JmmE)n*@TV^LT*es|ORB#~;(bqe<9LY%pCNY2%kMbxJVeo`tlD zXYk8q>@EVEGgyrAjC$#uFnNEb`I@j|1sUR0BKiG7L2KNsN+7XC((GPkOpZm8o3T}P zV{WeRlgMSKksrhNlyjPL=7#J;IzSt}m`VA$mLGK}(fRH_+M`Vf&iynpvx-&2EaI%c zeOue*$y%6BaKDO;K-cRT0gHzc6>re%4J&UGO^TFl?JA;*TZ)q;_rorkn-pwEc@8ZM zdYWDnGufkxL+)=V7mLU&bg^?kKcCBD<_j3GM<-P*Mhi8qj&JZyKe_$1r#G<<;WsJ zxnD$mnb*^z6VI8OD$AQ3Me9B7sinB7ZbkqUk41`CHk&sC<) z-)g!%FFVsiRNXt%vshhg8@{|gZKz%ANGm3IWTI-#(UXs4J%YF3H1c#F_<{PF(=YfcX3<*Wjpel3U+-ih5c66(tz-MI!TnJ$NczSO`hshI z-@}zI^NIbEKsPiBy>(V+4B;(ioD|oh<>>mx;KPJr&)z^>yi+T>OF?b1sG5yGCDo1^ zYp$_&G8C+8iTAdOewH;ZeJ#0U$0v#eePT@D<1FY@O>?q)R+{yxGt3+nS~DRl&WH3l zxomq8v(@gIp>?BNVyh+HH@6T!;WwTJi3MdGq*BJvPeh?pn>b(cSuRR#3|$2Dd3jp% zavOTA2vFJ`k8tg~4aBWBj($H{DOCgQ&B%9~(uv@nOc1v!{GF1dt;yA=A;qABe%Iae zfn1WPa2E|Qmahw1T6*!NAU*GlMaP%mUuKU{-i#-Go?Gx9%_*LHKeHQ1K+UYy`1l2x z1!qht&Ch-ul8_iF`rN+!rRQ7ux3$LDdf&B4%aI-2V1`s*dEE`SV3>D+r*)RHe}Dny zBeP>yDrZQWu>StW%^R;H3aO0vtX(S~ojKy}560_`Eym`*-iFAPI#NzqoN)o^2wpFfRb_dHb8AiF~ z5|mELCLBahup&2Wc@Bz8%segj15hSEJ}$}{6Az+=b0S!eIcGQy5;w5=x@_$H9RDAh z54rQOP6Fmtf6l+S_@4; + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/wa/workloads/benchmarkpi/uiauto/project.properties b/wa/workloads/benchmarkpi/uiauto/project.properties new file mode 100644 index 00000000..a3ee5ab6 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/project.properties @@ -0,0 +1,14 @@ +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. +# +# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): +#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt + +# Project target. +target=android-17 diff --git a/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java b/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java new file mode 100644 index 00000000..097578f2 --- /dev/null +++ b/wa/workloads/benchmarkpi/uiauto/src/com/arm/wa/uiauto/UiAutomation.java @@ -0,0 +1,62 @@ +/* Copyright 2013-2015 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wa.uiauto.benchmarkpi; + +import android.app.Activity; +import android.os.Bundle; +import android.util.Log; + +// Import the uiautomator libraries +import com.android.uiautomator.core.UiObject; +import com.android.uiautomator.core.UiObjectNotFoundException; +import com.android.uiautomator.core.UiScrollable; +import com.android.uiautomator.core.UiSelector; +import com.android.uiautomator.testrunner.UiAutomatorTestCase; + +import com.arm.wa.uiauto.BaseUiAutomation; + +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "benchmarkpi"; + + public void runWorkload() throws Exception { + startTest(); + waitForResults(); + } + + public void extractResults() throws Exception { + UiSelector selector = new UiSelector(); + UiObject resultsText = new UiObject(selector.textContains("You calculated Pi in") + .className("android.widget.TextView")); + Log.v(TAG, resultsText.getText()); + } + + public void startTest() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject benchButton = new UiObject(selector.text("Benchmark my Android!") + .className("android.widget.Button")); + benchButton.click(); + } + + public void waitForResults() throws Exception{ + UiSelector selector = new UiSelector(); + UiObject submitButton = new UiObject(selector.text("Submit") + .className("android.widget.Button")); + submitButton.waitForExists(10 * 1000); + } + +}