mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-02-20 20:09:11 +00:00
Streamline resource resolution mechanics.
This commit is contained in:
Sergei Trofimov
parent
1d0db35e04
commit
15886ffa29
@ -12,4 +12,6 @@ from wa.framework.instrumentation import (Instrument, very_slow, slow, normal, f
|
||||
very_fast)
|
||||
from wa.framework.plugin import Plugin, Parameter
|
||||
from wa.framework.processor import ResultProcessor
|
||||
from wa.framework.resource import (NO_ONE, JarFile, ApkFile, ReventFile, File,
|
||||
Executable)
|
||||
from wa.framework.workload import Workload
|
||||
|
||||
@ -548,11 +548,11 @@ class MetaConfiguration(Configuration):
|
||||
|
||||
plugin_packages = [
|
||||
'wa.commands',
|
||||
'wa.workloads',
|
||||
'wa.framework.getters',
|
||||
'wa.framework.target.descriptor',
|
||||
'wa.instrumentation',
|
||||
'wa.processors',
|
||||
'wa.framework.target.descriptor',
|
||||
'wa.framework.resource_getters',
|
||||
'wa.workloads',
|
||||
]
|
||||
|
||||
config_points = [
|
||||
|
||||
@ -95,6 +95,9 @@ class PluginCache(object):
|
||||
def is_global_alias(self, name):
|
||||
return name in self._list_of_global_aliases
|
||||
|
||||
def list_plugins(self, kind=None):
|
||||
return self.loader.list_plugins(kind)
|
||||
|
||||
def get_plugin_config(self, plugin_name, generic_name=None):
|
||||
config = obj_dict(not_in_dict=['name'])
|
||||
config.name = plugin_name
|
||||
|
||||
@ -98,7 +98,7 @@ class ExecutionContext(object):
|
||||
self.run_state = output.state
|
||||
self.target_info = self.tm.get_target_info()
|
||||
self.logger.debug('Loading resource discoverers')
|
||||
self.resolver = ResourceResolver(cm)
|
||||
self.resolver = ResourceResolver(cm.plugin_cache)
|
||||
self.resolver.load()
|
||||
self.job_queue = None
|
||||
self.completed_jobs = None
|
||||
|
||||
102
wa/framework/getters.py
Normal file
102
wa/framework/getters.py
Normal file
@ -0,0 +1,102 @@
|
||||
# Copyright 2013-2015 ARM Limited
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
"""
|
||||
This module contains the standard set of resource getters used by Workload Automation.
|
||||
|
||||
"""
|
||||
import httplib
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
from devlib.utils.android import ApkInfo
|
||||
|
||||
from wa import Parameter, settings, __file__ as __base_filepath
|
||||
from wa.framework.resource import ResourceGetter, SourcePriority, NO_ONE
|
||||
from wa.framework.exception import ResourceError
|
||||
from wa.utils.misc import (ensure_directory_exists as _d,
|
||||
ensure_file_directory_exists as _f, sha256, urljoin)
|
||||
from wa.utils.types import boolean, caseless_string
|
||||
|
||||
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
logger = logging.getLogger('resource')
|
||||
|
||||
|
||||
def get_by_extension(path, ext):
|
||||
if not ext.startswith('.'):
|
||||
ext = '.' + ext
|
||||
ext = caseless_string(ext)
|
||||
|
||||
found = []
|
||||
for entry in os.listdir(path):
|
||||
entry_ext = os.path.splitext(entry)
|
||||
if entry_ext == ext:
|
||||
found.append(os.path.join(path, entry))
|
||||
return found
|
||||
|
||||
|
||||
def get_generic_resource(resource, files):
|
||||
matches = []
|
||||
for f in files:
|
||||
if resource.match(f):
|
||||
matches.append(f)
|
||||
if not matches:
|
||||
return None
|
||||
if len(matches) > 1:
|
||||
msg = 'Multiple matches for {}: {}'
|
||||
return ResourceError(msg.format(resource, matches))
|
||||
return matches[0]
|
||||
|
||||
|
||||
class Package(ResourceGetter):
|
||||
|
||||
name = 'package'
|
||||
|
||||
def register(self, resolver):
|
||||
resolver.register(self.get, SourcePriority.package)
|
||||
|
||||
def get(self, resource):
|
||||
if resource.owner == NO_ONE:
|
||||
basepath = os.path.join(os.path.dirname(__base_filepath), 'assets')
|
||||
else:
|
||||
modname = resource.owner.__module__
|
||||
basepath = os.path.dirname(sys.modules[modname].__file__)
|
||||
|
||||
if resource.kind == 'file':
|
||||
path = os.path.join(basepath, resource.path)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
elif resource.kind == 'executable':
|
||||
path = os.path.join(basepath, 'bin', resource.abi, resource.filename)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
elif resource.kind in ['apk', 'jar', 'revent']:
|
||||
files = get_by_extension(basepath, resource.kind)
|
||||
return get_generic_resource(resource, files)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@ -12,52 +12,29 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
import shutil
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
from devlib.utils.android import ApkInfo
|
||||
|
||||
from wa.framework import pluginloader
|
||||
from wa.framework.plugin import Plugin, Parameter
|
||||
from wa.framework.exception import ResourceError
|
||||
from wa.framework.configuration import settings
|
||||
from wa.utils.misc import ensure_directory_exists as _d
|
||||
from wa.utils.types import boolean
|
||||
from wa.utils.types import prioritylist
|
||||
from wa.utils import log
|
||||
from wa.utils.misc import ensure_directory_exists as _d, get_object_name
|
||||
from wa.utils.types import boolean, prioritylist, enum
|
||||
|
||||
|
||||
class GetterPriority(object):
|
||||
"""
|
||||
Enumerates standard ResourceGetter priorities. In general, getters should
|
||||
register under one of these, rather than specifying other priority values.
|
||||
|
||||
|
||||
:cached: The cached version of the resource. Look here first. This
|
||||
priority also implies
|
||||
that the resource at this location is a "cache" and is not
|
||||
the only version of the resource, so it may be cleared without
|
||||
losing access to the resource.
|
||||
:preferred: Take this resource in favour of the environment resource.
|
||||
:environment: Found somewhere under ~/.workload_automation/ or equivalent,
|
||||
or from environment variables, external configuration
|
||||
files, etc. These will override resource supplied with
|
||||
the package.
|
||||
:external_package: Resource provided by another package. :package:
|
||||
Resource provided with the package. :remote:
|
||||
Resource will be downloaded from a remote location
|
||||
(such as an HTTP server or a samba share). Try this
|
||||
only if no other getter was successful.
|
||||
|
||||
"""
|
||||
cached = 20
|
||||
preferred = 10
|
||||
environment = 0
|
||||
external_package = -5
|
||||
package = -10
|
||||
remote = -20
|
||||
SourcePriority = enum(['package', 'remote', 'lan', 'external_package', 'local',
|
||||
'perferred'], start=0, step=10)
|
||||
|
||||
|
||||
class __NullOwner(object):
|
||||
@ -77,6 +54,7 @@ class __NullOwner(object):
|
||||
|
||||
NO_ONE = __NullOwner()
|
||||
|
||||
|
||||
class Resource(object):
|
||||
"""
|
||||
Represents a resource that needs to be resolved. This can be pretty much
|
||||
@ -88,96 +66,106 @@ class Resource(object):
|
||||
|
||||
"""
|
||||
|
||||
name = None
|
||||
kind = None
|
||||
|
||||
def __init__(self, owner):
|
||||
def __init__(self, owner=NO_ONE):
|
||||
self.owner = owner
|
||||
|
||||
def delete(self, instance):
|
||||
"""
|
||||
Delete an instance of this resource type. This must be implemented
|
||||
by the concrete subclasses based on what the resource looks like,
|
||||
e.g. deleting a file or a directory tree, or removing an entry from
|
||||
a database.
|
||||
|
||||
:note: Implementation should *not* contain any logic for deciding
|
||||
whether or not a resource should be deleted, only the actual
|
||||
deletion. The assumption is that if this method is invoked,
|
||||
then the decision has already been made.
|
||||
|
||||
"""
|
||||
def match(self, path):
|
||||
raise NotImplementedError()
|
||||
|
||||
def __str__(self):
|
||||
return '<{}\'s {}>'.format(self.owner, self.name)
|
||||
|
||||
|
||||
class FileResource(Resource):
|
||||
"""
|
||||
Base class for all resources that are a regular file in the
|
||||
file system.
|
||||
class File(Resource):
|
||||
|
||||
"""
|
||||
|
||||
def delete(self, instance):
|
||||
os.remove(instance)
|
||||
|
||||
|
||||
class File(FileResource):
|
||||
|
||||
name = 'file'
|
||||
|
||||
def __init__(self, owner, path, url=None):
|
||||
super(File, self).__init__(owner)
|
||||
self.path = path
|
||||
self.url = url
|
||||
|
||||
def __str__(self):
|
||||
return '<{}\'s {} {}>'.format(self.owner, self.name, self.path or self.url)
|
||||
|
||||
|
||||
class PluginAsset(File):
|
||||
|
||||
name = 'plugin_asset'
|
||||
kind = 'file'
|
||||
|
||||
def __init__(self, owner, path):
|
||||
super(PluginAsset, self).__init__(owner, os.path.join(owner.name, path))
|
||||
super(File, self).__init__(owner)
|
||||
self.path = path
|
||||
|
||||
|
||||
class Executable(FileResource):
|
||||
|
||||
name = 'executable'
|
||||
|
||||
def __init__(self, owner, platform, filename):
|
||||
super(Executable, self).__init__(owner)
|
||||
self.platform = platform
|
||||
self.filename = filename
|
||||
def match(self, path):
|
||||
return self.path == path
|
||||
|
||||
def __str__(self):
|
||||
return '<{}\'s {} {}>'.format(self.owner, self.platform, self.filename)
|
||||
return '<{}\'s {} {}>'.format(self.owner, self.kind, self.path)
|
||||
|
||||
class ReventFile(FileResource):
|
||||
|
||||
name = 'revent'
|
||||
class Executable(Resource):
|
||||
|
||||
def __init__(self, owner, stage):
|
||||
kind = 'executable'
|
||||
|
||||
def __init__(self, owner, abi, filename):
|
||||
super(Executable, self).__init__(owner)
|
||||
self.abi = abi
|
||||
self.filename = filename
|
||||
|
||||
def match(self, path):
|
||||
return self.filename == os.path.basename(path)
|
||||
|
||||
def __str__(self):
|
||||
return '<{}\'s {} {}>'.format(self.owner, self.abi, self.filename)
|
||||
|
||||
|
||||
class ReventFile(Resource):
|
||||
|
||||
kind = 'revent'
|
||||
|
||||
def __init__(self, owner, stage, target):
|
||||
super(ReventFile, self).__init__(owner)
|
||||
self.stage = stage
|
||||
self.target = target
|
||||
|
||||
def match(self, path):
|
||||
filename = os.path.basename(path)
|
||||
parts = filename.split('.')
|
||||
if len(parts) > 2:
|
||||
target, stage = parts[:2]
|
||||
return target == self.target and stage == self.stage
|
||||
else:
|
||||
stage = parts[0]
|
||||
return stage == self.stage
|
||||
|
||||
|
||||
class JarFile(FileResource):
|
||||
class JarFile(Resource):
|
||||
|
||||
name = 'jar'
|
||||
kind = 'jar'
|
||||
|
||||
def match(self, path):
|
||||
# An owner always has at most one jar file, so
|
||||
# always match
|
||||
return True
|
||||
|
||||
|
||||
class ApkFile(FileResource):
|
||||
class ApkFile(Resource):
|
||||
|
||||
name = 'apk'
|
||||
kind = 'apk'
|
||||
|
||||
def __init__(self, owner, version):
|
||||
def __init__(self, owner, variant=None, version=None):
|
||||
super(ApkFile, self).__init__(owner)
|
||||
self.variant = variant
|
||||
self.version = version
|
||||
|
||||
def match(self, path):
|
||||
name_matches = True
|
||||
version_matches = True
|
||||
if self.version is not None:
|
||||
version_matches = apk_version_matches(path, self.version)
|
||||
if self.variant is not None:
|
||||
name_matches = file_name_matches(path, self.variant)
|
||||
return name_matches and version_matches:
|
||||
|
||||
def __str__(self):
|
||||
text = '<{}\'s apk'.format(self.owner)
|
||||
if self.variant:
|
||||
text += ' {}'.format(self.variant)
|
||||
if self.version:
|
||||
text += ' {}'.format(self.version)
|
||||
text += '>'
|
||||
return text
|
||||
|
||||
|
||||
class ResourceGetter(Plugin):
|
||||
"""
|
||||
@ -192,10 +180,6 @@ class ResourceGetter(Plugin):
|
||||
|
||||
:name: Name that uniquely identifies this getter. Must be set by any
|
||||
concrete subclass.
|
||||
:resource_type: Identifies resource type(s) that this getter can
|
||||
handle. This must be either a string (for a single type)
|
||||
or a list of strings for multiple resource types. This
|
||||
must be set by any concrete subclass.
|
||||
:priority: Priority with which this getter will be invoked. This should
|
||||
be one of the standard priorities specified in
|
||||
``GetterPriority`` enumeration. If not set, this will default
|
||||
@ -205,74 +189,12 @@ class ResourceGetter(Plugin):
|
||||
|
||||
name = None
|
||||
kind = 'resource_getter'
|
||||
resource_type = None
|
||||
priority = GetterPriority.environment
|
||||
|
||||
def __init__(self, resolver, **kwargs):
|
||||
super(ResourceGetter, self).__init__(**kwargs)
|
||||
self.resolver = resolver
|
||||
|
||||
def register(self):
|
||||
"""
|
||||
Registers with a resource resolver. Concrete implementations must
|
||||
override this to invoke ``self.resolver.register()`` method to register
|
||||
``self`` for specific resource types.
|
||||
|
||||
"""
|
||||
if self.resource_type is None:
|
||||
message = 'No resource type specified for {}'
|
||||
raise ValueError(message.format(self.name))
|
||||
elif isinstance(self.resource_type, list):
|
||||
for rt in self.resource_type:
|
||||
self.resolver.register(self, rt, self.priority)
|
||||
else:
|
||||
self.resolver.register(self, self.resource_type, self.priority)
|
||||
|
||||
def unregister(self):
|
||||
"""Unregister from a resource resolver."""
|
||||
if self.resource_type is None:
|
||||
message = 'No resource type specified for {}'
|
||||
raise ValueError(message.format(self.name))
|
||||
elif isinstance(self.resource_type, list):
|
||||
for rt in self.resource_type:
|
||||
self.resolver.unregister(self, rt)
|
||||
else:
|
||||
self.resolver.unregister(self, self.resource_type)
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
"""
|
||||
This will get invoked by the resolver when attempting to resolve a
|
||||
resource, passing in the resource to be resolved as the first
|
||||
parameter. Any additional parameters would be specific to a particular
|
||||
resource type.
|
||||
|
||||
This method will only be invoked for resource types that the getter has
|
||||
registered for.
|
||||
|
||||
:param resource: an instance of :class:`wlauto.core.resource.Resource`.
|
||||
|
||||
:returns: Implementations of this method must return either the
|
||||
discovered resource or ``None`` if the resource could not
|
||||
be discovered.
|
||||
|
||||
"""
|
||||
def register(self, resolver):
|
||||
raise NotImplementedError()
|
||||
|
||||
def delete(self, resource, *args, **kwargs):
|
||||
"""
|
||||
Delete the resource if it is discovered. All arguments are passed to a
|
||||
call to``self.get()``. If that call returns a resource, it is deleted.
|
||||
|
||||
:returns: ``True`` if the specified resource has been discovered
|
||||
and deleted, and ``False`` otherwise.
|
||||
|
||||
"""
|
||||
discovered = self.get(resource, *args, **kwargs)
|
||||
if discovered:
|
||||
resource.delete(discovered)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
def initialize(self):
|
||||
pass
|
||||
|
||||
def __str__(self):
|
||||
return '<ResourceGetter {}>'.format(self.name)
|
||||
@ -285,23 +207,31 @@ class ResourceResolver(object):
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, config):
|
||||
def __init__(self, loader=pluginloader):
|
||||
self.loader = loader
|
||||
self.logger = logging.getLogger('resolver')
|
||||
self.getters = defaultdict(prioritylist)
|
||||
self.config = config
|
||||
self.getters = []
|
||||
self.sources = prioritylist()
|
||||
|
||||
def load(self):
|
||||
"""
|
||||
Discover getters under the specified source. The source could
|
||||
be either a python package/module or a path.
|
||||
for gettercls in self.loader.list_plugins('resource_getter'):
|
||||
self.logger.debug('Loading getter {}'.format(gettercls.name))
|
||||
getter = self.loader.get_plugin(name=gettercls.name,
|
||||
kind="resource_getter")
|
||||
log.indent()
|
||||
try:
|
||||
getter.initialize()
|
||||
getter.register(self)
|
||||
finally:
|
||||
log.dedent()
|
||||
self.getters.append(getter)
|
||||
|
||||
"""
|
||||
def register(self, source, priority=SourcePriority.local):
|
||||
msg = 'Registering "{}" with priority "{}"'
|
||||
self.logger.debug(msg.format(get_object_name(source), priority))
|
||||
self.sources.add(source, priority)
|
||||
|
||||
for rescls in pluginloader.list_resource_getters():
|
||||
getter = self.config.get_plugin(name=rescls.name, kind="resource_getter", resolver=self)
|
||||
getter.register()
|
||||
|
||||
def get(self, resource, strict=True, *args, **kwargs):
|
||||
def get(self, resource, strict=True):
|
||||
"""
|
||||
Uses registered getters to attempt to discover a resource of the specified
|
||||
kind and matching the specified criteria. Returns path to the resource that
|
||||
@ -311,11 +241,13 @@ class ResourceResolver(object):
|
||||
|
||||
"""
|
||||
self.logger.debug('Resolving {}'.format(resource))
|
||||
for getter in self.getters[resource.name]:
|
||||
self.logger.debug('Trying {}'.format(getter))
|
||||
result = getter.get(resource, *args, **kwargs)
|
||||
for source in self.sources:
|
||||
source_name = get_object_name(source)
|
||||
self.logger.debug('Trying {}'.format(source_name))
|
||||
result = source(resource)
|
||||
if result is not None:
|
||||
self.logger.debug('Resource {} found using {}:'.format(resource, getter))
|
||||
msg = 'Resource {} found using {}:'
|
||||
self.logger.debug(msg.format(resource, source_name))
|
||||
self.logger.debug('\t{}'.format(result))
|
||||
return result
|
||||
if strict:
|
||||
@ -323,61 +255,19 @@ class ResourceResolver(object):
|
||||
self.logger.debug('Resource {} not found.'.format(resource))
|
||||
return None
|
||||
|
||||
def register(self, getter, kind, priority=0):
|
||||
"""
|
||||
Register the specified resource getter as being able to discover a resource
|
||||
of the specified kind with the specified priority.
|
||||
|
||||
This method would typically be invoked by a getter inside its __init__.
|
||||
The idea being that getters register themselves for resources they know
|
||||
they can discover.
|
||||
|
||||
*priorities*
|
||||
|
||||
getters that are registered with the highest priority will be invoked first. If
|
||||
multiple getters are registered under the same priority, they will be invoked
|
||||
in the order they were registered (i.e. in the order they were discovered). This is
|
||||
essentially non-deterministic.
|
||||
|
||||
Generally getters that are more likely to find a resource, or would find a
|
||||
"better" version of the resource should register with higher (positive) priorities.
|
||||
Fall-back getters that should only be invoked if a resource is not found by usual
|
||||
means should register with lower (negative) priorities.
|
||||
|
||||
"""
|
||||
self.logger.debug('Registering {} for {} resources'.format(getter.name, kind))
|
||||
self.getters[kind].add(getter, priority)
|
||||
|
||||
def unregister(self, getter, kind):
|
||||
"""
|
||||
Unregister a getter that has been registered earlier.
|
||||
|
||||
"""
|
||||
self.logger.debug('Unregistering {}'.format(getter.name))
|
||||
try:
|
||||
self.getters[kind].remove(getter)
|
||||
except ValueError:
|
||||
raise ValueError('Resource getter {} is not installed.'.format(getter.name))
|
||||
|
||||
# Utility functions
|
||||
|
||||
def get_from_location_by_extension(resource, location, extension, version=None):
|
||||
found_files = glob.glob(os.path.join(location, '*.{}'.format(extension)))
|
||||
if version:
|
||||
found_files = [ff for ff in found_files
|
||||
if version.lower() in os.path.basename(ff).lower()]
|
||||
if len(found_files) == 1:
|
||||
return found_files[0]
|
||||
elif not found_files:
|
||||
return None
|
||||
else:
|
||||
raise ResourceError('More than one .{} found in {} for {}.'.format(extension,
|
||||
location,
|
||||
resource.owner.name))
|
||||
def apk_version_matches(path, version):
|
||||
info = ApkInfo(path)
|
||||
if info.version_name == version or info.version_code == version:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_owner_path(resource):
|
||||
if resource.owner is NO_ONE:
|
||||
return os.path.join(os.path.dirname(__base_filepath), 'common')
|
||||
else:
|
||||
return os.path.dirname(sys.modules[resource.owner.__module__].__file__)
|
||||
def file_name_matches(path, pattern):
|
||||
filename = os.path.basename(path)
|
||||
if pattern in filename:
|
||||
return True
|
||||
if re.search(pattern, filename):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@ -1,510 +0,0 @@
|
||||
# Copyright 2013-2015 ARM Limited
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
"""
|
||||
This module contains the standard set of resource getters used by Workload Automation.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import inspect
|
||||
import httplib
|
||||
import logging
|
||||
import json
|
||||
|
||||
import requests
|
||||
|
||||
from wa import Parameter, settings, __file__ as __base_filepath
|
||||
from wa.framework.resource import ResourceGetter, GetterPriority, NO_ONE
|
||||
from wa.framework.exception import ResourceError
|
||||
from wa.utils.misc import (ensure_directory_exists as _d,
|
||||
ensure_file_directory_exists as _f, sha256, urljoin)
|
||||
from wa.utils.types import boolean
|
||||
|
||||
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
class PackageFileGetter(ResourceGetter):
|
||||
|
||||
name = 'package_file'
|
||||
description = """
|
||||
Looks for exactly one file with the specified plugin in the owner's directory. If a version
|
||||
is specified on invocation of get, it will filter the discovered file based on that version.
|
||||
Versions are treated as case-insensitive.
|
||||
"""
|
||||
|
||||
plugin = None
|
||||
|
||||
def register(self):
|
||||
self.resolver.register(self, self.plugin, GetterPriority.package)
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
resource_dir = os.path.dirname(sys.modules[resource.owner.__module__].__file__)
|
||||
version = kwargs.get('version')
|
||||
return get_from_location_by_plugin(resource, resource_dir, self.plugin, version)
|
||||
|
||||
|
||||
class EnvironmentFileGetter(ResourceGetter):
|
||||
|
||||
name = 'environment_file'
|
||||
description = """Looks for exactly one file with the specified plugin in the owner's directory. If a version
|
||||
is specified on invocation of get, it will filter the discovered file based on that version.
|
||||
Versions are treated as case-insensitive."""
|
||||
|
||||
plugin = None
|
||||
|
||||
def register(self):
|
||||
self.resolver.register(self, self.plugin, GetterPriority.environment)
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
resource_dir = resource.owner.dependencies_directory
|
||||
|
||||
version = kwargs.get('version')
|
||||
return get_from_location_by_plugin(resource, resource_dir, self.plugin, version)
|
||||
|
||||
|
||||
class ReventGetter(ResourceGetter):
|
||||
"""Implements logic for identifying revent files."""
|
||||
|
||||
def get_base_location(self, resource):
|
||||
raise NotImplementedError()
|
||||
|
||||
def register(self):
|
||||
self.resolver.register(self, 'revent', GetterPriority.package)
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
filename = '.'.join([resource.owner.device.model, resource.stage, 'revent']).lower()
|
||||
location = _d(os.path.join(self.get_base_location(resource), 'revent_files'))
|
||||
for candidate in os.listdir(location):
|
||||
if candidate.lower() == filename.lower():
|
||||
return os.path.join(location, candidate)
|
||||
|
||||
|
||||
class PackageApkGetter(PackageFileGetter):
|
||||
name = 'package_apk'
|
||||
plugin = 'apk'
|
||||
|
||||
|
||||
class PackageJarGetter(PackageFileGetter):
|
||||
name = 'package_jar'
|
||||
plugin = 'jar'
|
||||
|
||||
|
||||
class PackageReventGetter(ReventGetter):
|
||||
|
||||
name = 'package_revent'
|
||||
|
||||
def get_base_location(self, resource):
|
||||
return get_owner_path(resource)
|
||||
|
||||
|
||||
class EnvironmentApkGetter(EnvironmentFileGetter):
|
||||
name = 'environment_apk'
|
||||
plugin = 'apk'
|
||||
|
||||
|
||||
class EnvironmentJarGetter(EnvironmentFileGetter):
|
||||
name = 'environment_jar'
|
||||
plugin = 'jar'
|
||||
|
||||
|
||||
class EnvironmentReventGetter(ReventGetter):
|
||||
|
||||
name = 'enviroment_revent'
|
||||
|
||||
def get_base_location(self, resource):
|
||||
return resource.owner.dependencies_directory
|
||||
|
||||
|
||||
class ExecutableGetter(ResourceGetter):
|
||||
|
||||
name = 'exe_getter'
|
||||
resource_type = 'executable'
|
||||
priority = GetterPriority.environment
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
if settings.assets_repository:
|
||||
path = os.path.join(settings.assets_repository, resource.platform, resource.filename)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
|
||||
|
||||
class PackageExecutableGetter(ExecutableGetter):
|
||||
|
||||
name = 'package_exe_getter'
|
||||
priority = GetterPriority.package
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
path = os.path.join(get_owner_path(resource), 'bin', resource.platform, resource.filename)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
|
||||
|
||||
class EnvironmentExecutableGetter(ExecutableGetter):
|
||||
|
||||
name = 'env_exe_getter'
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
paths = [
|
||||
os.path.join(resource.owner.dependencies_directory, 'bin',
|
||||
resource.platform, resource.filename),
|
||||
os.path.join(settings.user_directory, 'bin',
|
||||
resource.platform, resource.filename),
|
||||
]
|
||||
for path in paths:
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
|
||||
|
||||
class DependencyFileGetter(ResourceGetter):
|
||||
|
||||
name = 'filer'
|
||||
description = """
|
||||
Gets resources from the specified mount point. Copies them the local dependencies
|
||||
directory, and returns the path to the local copy.
|
||||
|
||||
"""
|
||||
resource_type = 'file'
|
||||
relative_path = '' # May be overridden by subclasses.
|
||||
|
||||
priority = GetterPriority.remote
|
||||
|
||||
parameters = [
|
||||
Parameter('mount_point', default='/', global_alias='remote_assets_path',
|
||||
description='Local mount point for the remote filer.'),
|
||||
]
|
||||
|
||||
def __init__(self, resolver, **kwargs):
|
||||
super(DependencyFileGetter, self).__init__(resolver, **kwargs)
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
force = kwargs.get('force')
|
||||
remote_path = os.path.join(self.mount_point, self.relative_path, resource.path)
|
||||
local_path = os.path.join(resource.owner.dependencies_directory, os.path.basename(resource.path))
|
||||
|
||||
if not os.path.isfile(local_path) or force:
|
||||
if not os.path.isfile(remote_path):
|
||||
return None
|
||||
self.logger.debug('Copying {} to {}'.format(remote_path, local_path))
|
||||
shutil.copy(remote_path, local_path)
|
||||
|
||||
return local_path
|
||||
|
||||
|
||||
class PackageCommonDependencyGetter(ResourceGetter):
|
||||
|
||||
name = 'packaged_common_dependency'
|
||||
resource_type = 'file'
|
||||
priority = GetterPriority.package - 1 # check after owner-specific locations
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
path = os.path.join(settings.package_directory, 'common', resource.path)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
|
||||
class EnvironmentCommonDependencyGetter(ResourceGetter):
|
||||
|
||||
name = 'environment_common_dependency'
|
||||
resource_type = 'file'
|
||||
priority = GetterPriority.environment - 1 # check after owner-specific locations
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
path = os.path.join(settings.dependencies_directory,
|
||||
os.path.basename(resource.path))
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
|
||||
class PackageDependencyGetter(ResourceGetter):
|
||||
|
||||
name = 'packaged_dependency'
|
||||
resource_type = 'file'
|
||||
priority = GetterPriority.package
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
owner_path = inspect.getfile(resource.owner.__class__)
|
||||
path = os.path.join(os.path.dirname(owner_path), resource.path)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
|
||||
class EnvironmentDependencyGetter(ResourceGetter):
|
||||
|
||||
name = 'environment_dependency'
|
||||
resource_type = 'file'
|
||||
priority = GetterPriority.environment
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
path = os.path.join(resource.owner.dependencies_directory, os.path.basename(resource.path))
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
|
||||
class PluginAssetGetter(DependencyFileGetter):
|
||||
|
||||
name = 'plugin_asset'
|
||||
resource_type = 'plugin_asset'
|
||||
|
||||
|
||||
class HttpGetter(ResourceGetter):
|
||||
|
||||
name = 'http_assets'
|
||||
description = """
|
||||
Downloads resources from a server based on an index fetched from the specified URL.
|
||||
|
||||
Given a URL, this will try to fetch ``<URL>/index.json``. The index file maps plugin
|
||||
names to a list of corresponing asset descriptons. Each asset description continas a path
|
||||
(relative to the base URL) of the resource and a SHA256 hash, so that this Getter can
|
||||
verify whether the resource on the remote has changed.
|
||||
|
||||
For example, let's assume we want to get the APK file for workload "foo", and that
|
||||
assets are hosted at ``http://example.com/assets``. This Getter will first try to
|
||||
donwload ``http://example.com/assests/index.json``. The index file may contian
|
||||
something like ::
|
||||
|
||||
{
|
||||
"foo": [
|
||||
{
|
||||
"path": "foo-app.apk",
|
||||
"sha256": "b14530bb47e04ed655ac5e80e69beaa61c2020450e18638f54384332dffebe86"
|
||||
},
|
||||
{
|
||||
"path": "subdir/some-other-asset.file",
|
||||
"sha256": "48d9050e9802246d820625717b72f1c2ba431904b8484ca39befd68d1dbedfff"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
This Getter will look through the list of assets for "foo" (in this case, two) check
|
||||
the paths until it finds one matching the resource (in this case, "foo-app.apk").
|
||||
Finally, it will try to dowload that file relative to the base URL and plugin name
|
||||
(in this case, "http://example.com/assets/foo/foo-app.apk"). The downloaded version
|
||||
will be cached locally, so that in the future, the getter will check the SHA256 hash
|
||||
of the local file against the one advertised inside index.json, and provided that hasn't
|
||||
changed, it won't try to download the file again.
|
||||
|
||||
"""
|
||||
priority = GetterPriority.remote
|
||||
resource_type = ['apk', 'file', 'jar', 'revent']
|
||||
|
||||
parameters = [
|
||||
Parameter('url', global_alias='remote_assets_url',
|
||||
description="""URL of the index file for assets on an HTTP server."""),
|
||||
Parameter('username',
|
||||
description="""User name for authenticating with assets URL"""),
|
||||
Parameter('password',
|
||||
description="""Password for authenticationg with assets URL"""),
|
||||
Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets',
|
||||
description="""If ``True``, will always attempt to fetch assets from the remote, even if
|
||||
a local cached copy is available."""),
|
||||
Parameter('chunk_size', kind=int, default=1024,
|
||||
description="""Chunk size for streaming large assets."""),
|
||||
]
|
||||
|
||||
def __init__(self, resolver, **kwargs):
|
||||
super(HttpGetter, self).__init__(resolver, **kwargs)
|
||||
self.index = None
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
if not resource.owner:
|
||||
return # TODO: add support for unowned resources
|
||||
if not self.index:
|
||||
self.index = self.fetch_index()
|
||||
asset = self.resolve_resource(resource)
|
||||
if not asset:
|
||||
return
|
||||
return self.download_asset(asset, resource.owner.name)
|
||||
|
||||
def fetch_index(self):
|
||||
if not self.url:
|
||||
return {}
|
||||
index_url = urljoin(self.url, 'index.json')
|
||||
response = self.geturl(index_url)
|
||||
if response.status_code != httplib.OK:
|
||||
message = 'Could not fetch "{}"; recieved "{} {}"'
|
||||
self.logger.error(message.format(index_url, response.status_code, response.reason))
|
||||
return {}
|
||||
return json.loads(response.content)
|
||||
|
||||
def download_asset(self, asset, owner_name):
|
||||
url = urljoin(self.url, owner_name, asset['path'])
|
||||
local_path = _f(os.path.join(settings.dependencies_directory, '__remote',
|
||||
owner_name, asset['path'].replace('/', os.sep)))
|
||||
if os.path.isfile(local_path) and not self.always_fetch:
|
||||
local_sha = sha256(local_path)
|
||||
if local_sha == asset['sha256']:
|
||||
self.logger.debug('Local SHA256 matches; not re-downloading')
|
||||
return local_path
|
||||
self.logger.debug('Downloading {}'.format(url))
|
||||
response = self.geturl(url, stream=True)
|
||||
if response.status_code != httplib.OK:
|
||||
message = 'Could not download asset "{}"; recieved "{} {}"'
|
||||
self.logger.warning(message.format(url, response.status_code, response.reason))
|
||||
return
|
||||
with open(local_path, 'wb') as wfh:
|
||||
for chunk in response.iter_content(chunk_size=self.chunk_size):
|
||||
wfh.write(chunk)
|
||||
return local_path
|
||||
|
||||
def geturl(self, url, stream=False):
|
||||
if self.username:
|
||||
auth = (self.username, self.password)
|
||||
else:
|
||||
auth = None
|
||||
return requests.get(url, auth=auth, stream=stream)
|
||||
|
||||
def resolve_resource(self, resource):
|
||||
assets = self.index.get(resource.owner.name, {})
|
||||
if not assets:
|
||||
return {}
|
||||
if resource.name in ['apk', 'jar']:
|
||||
paths = [a['path'] for a in assets]
|
||||
version = getattr(resource, 'version', None)
|
||||
found = get_from_list_by_plugin(resource, paths, resource.name, version)
|
||||
if found:
|
||||
for a in assets:
|
||||
if a['path'] == found:
|
||||
return a
|
||||
elif resource.name == 'revent':
|
||||
filename = '.'.join([resource.owner.device.name, resource.stage, 'revent']).lower()
|
||||
for asset in assets:
|
||||
pathname = os.path.basename(asset['path']).lower()
|
||||
if pathname == filename:
|
||||
return asset
|
||||
else: # file
|
||||
for asset in assets:
|
||||
if asset['path'].lower() == resource.path.lower():
|
||||
return asset
|
||||
|
||||
|
||||
class RemoteFilerGetter(ResourceGetter):
|
||||
|
||||
name = 'filer_assets'
|
||||
description = """
|
||||
Finds resources on a (locally mounted) remote filer and caches them locally.
|
||||
|
||||
This assumes that the filer is mounted on the local machine (e.g. as a samba share).
|
||||
|
||||
"""
|
||||
priority = GetterPriority.remote
|
||||
resource_type = ['apk', 'file', 'jar', 'revent']
|
||||
|
||||
parameters = [
|
||||
Parameter('remote_path', global_alias='remote_assets_path', default='',
|
||||
description="""Path, on the local system, where the assets are located."""),
|
||||
Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets',
|
||||
description="""If ``True``, will always attempt to fetch assets from the remote, even if
|
||||
a local cached copy is available."""),
|
||||
]
|
||||
|
||||
def get(self, resource, **kwargs):
|
||||
version = kwargs.get('version')
|
||||
if resource.owner:
|
||||
remote_path = os.path.join(self.remote_path, resource.owner.name)
|
||||
local_path = os.path.join(settings.user_directory, '__filer', resource.owner.dependencies_directory)
|
||||
return self.try_get_resource(resource, version, remote_path, local_path)
|
||||
else:
|
||||
result = None
|
||||
for entry in os.listdir(remote_path):
|
||||
remote_path = os.path.join(self.remote_path, entry)
|
||||
local_path = os.path.join(settings.user_directory, '__filer', settings.dependencies_directory, entry)
|
||||
result = self.try_get_resource(resource, version, remote_path, local_path)
|
||||
if result:
|
||||
break
|
||||
return result
|
||||
|
||||
def try_get_resource(self, resource, version, remote_path, local_path):
|
||||
if not self.always_fetch:
|
||||
result = self.get_from(resource, version, local_path)
|
||||
if result:
|
||||
return result
|
||||
if remote_path:
|
||||
# Didn't find it cached locally; now check the remoted
|
||||
result = self.get_from(resource, version, remote_path)
|
||||
if not result:
|
||||
return result
|
||||
else: # remote path is not set
|
||||
return None
|
||||
# Found it remotely, cache locally, then return it
|
||||
local_full_path = os.path.join(_d(local_path), os.path.basename(result))
|
||||
self.logger.debug('cp {} {}'.format(result, local_full_path))
|
||||
shutil.copy(result, local_full_path)
|
||||
return local_full_path
|
||||
|
||||
def get_from(self, resource, version, location): # pylint: disable=no-self-use
|
||||
if resource.name in ['apk', 'jar']:
|
||||
return get_from_location_by_plugin(resource, location, resource.name, version)
|
||||
elif resource.name == 'file':
|
||||
filepath = os.path.join(location, resource.path)
|
||||
if os.path.exists(filepath):
|
||||
return filepath
|
||||
elif resource.name == 'revent':
|
||||
filename = '.'.join([resource.owner.device.model, resource.stage, 'revent']).lower()
|
||||
alternate_location = os.path.join(location, 'revent_files')
|
||||
# There tends to be some confusion as to where revent files should
|
||||
# be placed. This looks both in the plugin's directory, and in
|
||||
# 'revent_files' subdirectory under it, if it exists.
|
||||
if os.path.isdir(alternate_location):
|
||||
for candidate in os.listdir(alternate_location):
|
||||
if candidate.lower() == filename.lower():
|
||||
return os.path.join(alternate_location, candidate)
|
||||
if os.path.isdir(location):
|
||||
for candidate in os.listdir(location):
|
||||
if candidate.lower() == filename.lower():
|
||||
return os.path.join(location, candidate)
|
||||
else:
|
||||
raise ValueError('Unexpected resource type: {}'.format(resource.name))
|
||||
|
||||
|
||||
# Utility functions
|
||||
|
||||
def get_from_location_by_plugin(resource, location, plugin, version=None):
|
||||
try:
|
||||
found_files = [os.path.join(location, f) for f in os.listdir(location)]
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
return get_from_list_by_plugin(resource, found_files, plugin, version)
|
||||
except ResourceError:
|
||||
raise ResourceError('More than one .{} found in {} for {}.'.format(plugin,
|
||||
location,
|
||||
resource.owner.name))
|
||||
|
||||
|
||||
def get_from_list_by_plugin(resource, filelist, plugin, version=None):
|
||||
filelist = [ff for ff in filelist
|
||||
if os.path.splitext(ff)[1].lower().endswith(plugin)]
|
||||
if version:
|
||||
filelist = [ff for ff in filelist if version.lower() in os.path.basename(ff).lower()]
|
||||
if len(filelist) == 1:
|
||||
return filelist[0]
|
||||
elif not filelist:
|
||||
return None
|
||||
else:
|
||||
raise ResourceError('More than one .{} found in {} for {}.'.format(plugin,
|
||||
filelist,
|
||||
resource.owner.name))
|
||||
|
||||
|
||||
def get_owner_path(resource):
|
||||
if resource.owner is NO_ONE:
|
||||
return os.path.join(os.path.dirname(__base_filepath), 'common')
|
||||
else:
|
||||
return os.path.dirname(sys.modules[resource.owner.__module__].__file__)
|
||||
@ -583,3 +583,17 @@ def merge_dicts_simple(base, other):
|
||||
def touch(path):
|
||||
with open(path, 'w'):
|
||||
pass
|
||||
|
||||
|
||||
def get_object_name(obj):
|
||||
if hasattr(obj, 'name'):
|
||||
return obj.name
|
||||
elif hasattr(obj,'func_name'):
|
||||
return obj.func_name
|
||||
elif hasattr(obj, 'im_func'):
|
||||
return '{}.{}'.format(obj.im_class.__name__, obj.im_func.func_name)
|
||||
elif hasattr(obj, '__name__'):
|
||||
return obj.__name__
|
||||
elif hasattr(obj, '__class__'):
|
||||
return obj.__class__.__name__
|
||||
return None
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user