From 1425a6f6c928db10a5513547d23fcf533c12ebf2 Mon Sep 17 00:00:00 2001 From: Marc Bonnici Date: Mon, 1 Jun 2020 10:35:18 +0100 Subject: [PATCH] Implement caching of ApkInfo Allow caching of ApkInfo to prevent the requirement of re-parsing of APK files. --- wa/framework/configuration/core.py | 4 ++ wa/framework/resource.py | 19 +++----- wa/framework/workload.py | 7 ++- wa/instruments/misc.py | 7 ++- wa/utils/android.py | 76 ++++++++++++++++++++++++++++-- 5 files changed, 87 insertions(+), 26 deletions(-) diff --git a/wa/framework/configuration/core.py b/wa/framework/configuration/core.py index 38eb13a1..50ba6edc 100644 --- a/wa/framework/configuration/core.py +++ b/wa/framework/configuration/core.py @@ -538,6 +538,10 @@ class MetaConfiguration(Configuration): def target_info_cache_file(self): return os.path.join(self.cache_directory, 'targets.json') + @property + def apk_info_cache_file(self): + return os.path.join(self.cache_directory, 'apk_info.json') + def __init__(self, environ=None): super(MetaConfiguration, self).__init__() if environ is None: diff --git a/wa/framework/resource.py b/wa/framework/resource.py index 8221b9f4..485a86e7 100644 --- a/wa/framework/resource.py +++ b/wa/framework/resource.py @@ -16,16 +16,14 @@ import logging import os import re -from devlib.utils.android import ApkInfo - from wa.framework import pluginloader from wa.framework.plugin import Plugin from wa.framework.exception import ResourceError from wa.framework.configuration import settings from wa.utils import log +from wa.utils.android import get_cacheable_apk_info from wa.utils.misc import get_object_name from wa.utils.types import enum, list_or_string, prioritylist, version_tuple -from wa.utils.misc import lock_file SourcePriority = enum(['package', 'remote', 'lan', 'local', @@ -281,8 +279,7 @@ class ResourceResolver(object): def apk_version_matches(path, version): version = list_or_string(version) - with lock_file(path): - info = ApkInfo(path) + info = get_cacheable_apk_info(path) for v in version: if info.version_name == v or info.version_code == v: return True @@ -292,8 +289,7 @@ def apk_version_matches(path, version): def apk_version_matches_range(path, min_version=None, max_version=None): - with lock_file(path): - info = ApkInfo(path) + info = get_cacheable_apk_info(path) return range_version_matching(info.version_name, min_version, max_version) @@ -336,21 +332,18 @@ def file_name_matches(path, pattern): def uiauto_test_matches(path, uiauto): - with lock_file(path): - info = ApkInfo(path) + info = get_cacheable_apk_info(path) return uiauto == ('com.arm.wa.uiauto' in info.package) def package_name_matches(path, package): - with lock_file(path): - info = ApkInfo(path) + info = get_cacheable_apk_info(path) return info.package == package def apk_abi_matches(path, supported_abi, exact_abi=False): supported_abi = list_or_string(supported_abi) - with lock_file(path): - info = ApkInfo(path) + info = get_cacheable_apk_info(path) # If no native code present, suitable for all devices. if not info.native_code: return True diff --git a/wa/framework/workload.py b/wa/framework/workload.py index b78580cd..feaf63a9 100644 --- a/wa/framework/workload.py +++ b/wa/framework/workload.py @@ -22,8 +22,8 @@ try: except ImportError: from pipes import quote -from devlib.utils.android import ApkInfo +from wa.utils.android import get_cacheable_apk_info from wa.framework.plugin import TargetedPlugin, Parameter from wa.framework.resource import (ApkFile, ReventFile, File, loose_version_matching, @@ -523,7 +523,7 @@ class UiAutomatorGUI(object): def init_resources(self, resolver): self.uiauto_file = resolver.get(ApkFile(self.owner, uiauto=True)) if not self.uiauto_package: - uiauto_info = ApkInfo(self.uiauto_file) + uiauto_info = get_cacheable_apk_info(self.uiauto_file) self.uiauto_package = uiauto_info.package def init_commands(self): @@ -743,8 +743,7 @@ class PackageHandler(object): self.resolve_package_from_host(context) if self.apk_file: - with lock_file(self.apk_file): - self.apk_info = ApkInfo(self.apk_file) + self.apk_info = get_cacheable_apk_info(self.apk_file) else: if self.error_msg: raise WorkloadError(self.error_msg) diff --git a/wa/instruments/misc.py b/wa/instruments/misc.py index bfdc6e80..a2e1f9c4 100644 --- a/wa/instruments/misc.py +++ b/wa/instruments/misc.py @@ -32,16 +32,16 @@ import tarfile from subprocess import CalledProcessError from devlib.exception import TargetError -from devlib.utils.android import ApkInfo from wa import Instrument, Parameter, very_fast from wa.framework.exception import ConfigError from wa.framework.instrument import slow from wa.utils.diff import diff_sysfs_dirs, diff_interrupt_files -from wa.utils.misc import as_relative, lock_file +from wa.utils.misc import as_relative from wa.utils.misc import ensure_file_directory_exists as _f from wa.utils.misc import ensure_directory_exists as _d from wa.utils.types import list_of_strings +from wa.utils.android import get_cacheable_apk_info logger = logging.getLogger(__name__) @@ -244,8 +244,7 @@ class ApkVersion(Instrument): def setup(self, context): if hasattr(context.workload, 'apk_file'): - with lock_file(context.workload.apk_file): - self.apk_info = ApkInfo(context.workload.apk_file) + self.apk_info = get_cacheable_apk_info(context.workload.apk_file) else: self.apk_info = None diff --git a/wa/utils/android.py b/wa/utils/android.py index c3ce763c..b21f99f5 100644 --- a/wa/utils/android.py +++ b/wa/utils/android.py @@ -19,8 +19,7 @@ from datetime import datetime from devlib.utils.android import ApkInfo as _ApkInfo -from wa import settings -from wa.framework.exception import ConfigError +from wa.framework.configuration import settings from wa.utils.serializer import read_pod, write_pod, Podable from wa.utils.types import enum from wa.utils.misc import lock_file @@ -30,7 +29,10 @@ LogcatLogLevel = enum(['verbose', 'debug', 'info', 'warn', 'error', 'assert'], s log_level_map = ''.join(n[0].upper() for n in LogcatLogLevel.names) -logger = logging.getLogger('logcat') +logcat_logger = logging.getLogger('logcat') +apk_info_cache_logger = logging.getLogger('apk_info_cache') + +apk_info_cache = None class LogcatEvent(object): @@ -81,14 +83,15 @@ class LogcatParser(object): tag = (parts.pop(0) if parts else '').strip() except Exception as e: # pylint: disable=broad-except message = 'Invalid metadata for line:\n\t{}\n\tgot: "{}"' - logger.warning(message.format(line, e)) + logcat_logger.warning(message.format(line, e)) return None return LogcatEvent(timestamp, pid, tid, level, tag, message) +# pylint: disable=protected-access,attribute-defined-outside-init class ApkInfo(_ApkInfo, Podable): - # Implement ApkInfo as a Podable class. + '''Implement ApkInfo as a Podable class.''' _pod_serialization_version = 1 @@ -132,3 +135,66 @@ class ApkInfo(_ApkInfo, Podable): pod['_pod_version'] = pod.get('_pod_version', 1) return pod + +class ApkInfoCache: + + @staticmethod + def _check_env(): + if not os.path.exists(settings.cache_directory): + os.makedirs(settings.cache_directory) + + def __init__(self, path=settings.apk_info_cache_file): + self._check_env() + self.path = path + self.last_modified = None + self.cache = {} + self._update_cache() + + def store(self, apk_info, apk_id, overwrite=True): + self._update_cache() + if apk_id in self.cache and not overwrite: + raise ValueError('ApkInfo for {} is already in cache.'.format(apk_info.path)) + self.cache[apk_id] = apk_info.to_pod() + with lock_file(self.path): + write_pod(self.cache, self.path) + self.last_modified = os.stat(self.path) + + def get_info(self, key): + self._update_cache() + pod = self.cache.get(key) + + info = ApkInfo.from_pod(pod) if pod else None + return info + + def _update_cache(self): + if not os.path.exists(self.path): + return + if self.last_modified != os.stat(self.path): + apk_info_cache_logger.debug('Updating cache {}'.format(self.path)) + with lock_file(self.path): + self.cache = read_pod(self.path) + self.last_modified = os.stat(self.path) + + +def get_cacheable_apk_info(path): + # pylint: disable=global-statement + global apk_info_cache + if not path: + return + stat = os.stat(path) + modified = stat.st_mtime + apk_id = '{}-{}'.format(path, modified) + info = apk_info_cache.get_info(apk_id) + + if info: + msg = 'Using ApkInfo ({}) from cache'.format(info.package) + else: + with lock_file(path): + info = ApkInfo(path) + apk_info_cache.store(info, apk_id, overwrite=True) + msg = 'Storing ApkInfo ({}) in cache'.format(info.package) + apk_info_cache_logger.debug(msg) + return info + + +apk_info_cache = ApkInfoCache()