Streamline resource resolution mechanics.

wa/__init__.py

@@ -12,4 +12,6 @@ from wa.framework.instrumentation import (Instrument, very_slow, slow, normal, f
                                           very_fast)
 from wa.framework.plugin import Plugin, Parameter
 from wa.framework.processor import ResultProcessor
+from wa.framework.resource import (NO_ONE, JarFile, ApkFile, ReventFile, File,
+                                   Executable)
 from wa.framework.workload import Workload

wa/framework/configuration/core.py

@@ -548,11 +548,11 @@ class MetaConfiguration(Configuration):
 
     plugin_packages = [
         'wa.commands',
-        'wa.workloads',
+        'wa.framework.getters',
+        'wa.framework.target.descriptor',
         'wa.instrumentation',
         'wa.processors',
-        'wa.framework.target.descriptor',
+        'wa.workloads',
-        'wa.framework.resource_getters',
     ]
 
     config_points = [

wa/framework/configuration/plugin_cache.py

@@ -95,6 +95,9 @@ class PluginCache(object):
     def is_global_alias(self, name):
         return name in self._list_of_global_aliases
 
+    def list_plugins(self, kind=None):
+        return self.loader.list_plugins(kind)
+
     def get_plugin_config(self, plugin_name, generic_name=None):
         config = obj_dict(not_in_dict=['name'])
         config.name = plugin_name

wa/framework/execution.py

@@ -98,7 +98,7 @@ class ExecutionContext(object):
         self.run_state = output.state
         self.target_info = self.tm.get_target_info()
         self.logger.debug('Loading resource discoverers')
-        self.resolver = ResourceResolver(cm)
+        self.resolver = ResourceResolver(cm.plugin_cache)
         self.resolver.load()
         self.job_queue = None
         self.completed_jobs = None

wa/framework/getters.py

@@ -0,0 +1,102 @@
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""
+This module contains the standard set of resource getters used by Workload Automation.
+
+"""
+import httplib
+import inspect
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+
+import requests
+
+from devlib.utils.android import ApkInfo
+
+from wa import Parameter, settings, __file__ as __base_filepath
+from wa.framework.resource import ResourceGetter, SourcePriority, NO_ONE 
+from wa.framework.exception import ResourceError
+from wa.utils.misc import (ensure_directory_exists as _d, 
+                           ensure_file_directory_exists as _f, sha256, urljoin)
+from wa.utils.types import boolean, caseless_string
+
+
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+logger = logging.getLogger('resource')
+
+
+def get_by_extension(path, ext):
+    if not ext.startswith('.'):
+        ext = '.' + ext
+    ext = caseless_string(ext)
+
+    found = []
+    for entry in os.listdir(path):
+        entry_ext = os.path.splitext(entry)
+        if entry_ext == ext:
+            found.append(os.path.join(path, entry))
+    return found
+
+
+def get_generic_resource(resource, files):
+    matches = []
+    for f in files:
+        if resource.match(f):
+            matches.append(f)
+    if not matches:
+        return None
+    if len(matches) > 1:
+        msg = 'Multiple matches for {}: {}'
+        return ResourceError(msg.format(resource, matches))
+    return matches[0]
+
+
+class Package(ResourceGetter):
+
+    name = 'package'
+
+    def register(self, resolver):
+        resolver.register(self.get, SourcePriority.package)
+
+    def get(self, resource):
+        if resource.owner == NO_ONE:
+            basepath = os.path.join(os.path.dirname(__base_filepath), 'assets')
+        else:
+            modname = resource.owner.__module__
+            basepath  = os.path.dirname(sys.modules[modname].__file__)
+
+        if resource.kind == 'file':
+            path = os.path.join(basepath, resource.path)
+            if os.path.exists(path):
+                return path
+        elif resource.kind == 'executable':
+            path = os.path.join(basepath, 'bin', resource.abi, resource.filename)
+            if os.path.exists(path):
+                return path
+        elif resource.kind in ['apk', 'jar', 'revent']:
+            files = get_by_extension(basepath, resource.kind)
+            return get_generic_resource(resource, files)
+
+        return None
+
+

wa/framework/resource.py

@@ -12,52 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import os
-import sys
 import glob
-import shutil
 import inspect
 import logging
+import os
+import re
+import shutil
+import sys
 from collections import defaultdict
 
+from devlib.utils.android import ApkInfo
+
 from wa.framework import pluginloader
 from wa.framework.plugin import Plugin, Parameter
 from wa.framework.exception import ResourceError
 from wa.framework.configuration import settings
-from wa.utils.misc import ensure_directory_exists as _d
+from wa.utils import log
-from wa.utils.types import boolean
+from wa.utils.misc import ensure_directory_exists as _d, get_object_name
-from wa.utils.types import prioritylist
+from wa.utils.types import boolean, prioritylist, enum
 
 
-class GetterPriority(object):
-    """
-    Enumerates standard ResourceGetter priorities. In general, getters should
-    register under one of these, rather than specifying other priority values.
 
-
+SourcePriority = enum(['package', 'remote', 'lan', 'external_package', 'local',
-    :cached: The cached version of the resource. Look here first. This
+                       'perferred'], start=0, step=10)
-    priority also implies
-             that the resource at this location is a "cache" and is not
-             the only version of the resource, so it may be cleared without
-             losing access to the resource.
-    :preferred: Take this resource in favour of the environment resource.
-    :environment: Found somewhere under ~/.workload_automation/ or equivalent,
-                  or from environment variables, external configuration
-                  files, etc.  These will override resource supplied with
-                  the package.
-    :external_package: Resource provided by another package.  :package:
-                       Resource provided with the package.  :remote:
-                       Resource will be downloaded from a remote location
-                       (such as an HTTP server or a samba share). Try this
-                       only if no other getter was successful.
-
-    """
-    cached = 20
-    preferred = 10
-    environment = 0
-    external_package = -5
-    package = -10
-    remote = -20
 
 
 class __NullOwner(object):
@@ -77,6 +54,7 @@ class __NullOwner(object):
 
 NO_ONE = __NullOwner()
 
+
 class Resource(object):
     """
     Represents a resource that needs to be resolved. This can be pretty much
@@ -88,96 +66,106 @@ class Resource(object):
 
     """
 
-    name = None
+    kind = None
 
-    def __init__(self, owner):
+    def __init__(self, owner=NO_ONE):
         self.owner = owner
 
-    def delete(self, instance):
+    def match(self, path):
-        """
-        Delete an instance of this resource type. This must be implemented
-        by the concrete subclasses based on what the resource looks like,
-        e.g. deleting a file or a directory tree, or removing an entry from
-        a database.
-
-        :note: Implementation should *not* contain any logic for deciding
-               whether or not a resource should be deleted, only the actual
-               deletion. The assumption is that if this method is invoked,
-               then the decision has already been made.
-
-        """
         raise NotImplementedError()
 
     def __str__(self):
         return '<{}\'s {}>'.format(self.owner, self.name)
 
 
-class FileResource(Resource):
+class File(Resource):
-    """
-    Base class for all resources that are a regular file in the
-    file system.
 
-    """
+    kind = 'file'
-
-    def delete(self, instance):
-        os.remove(instance)
-
-
-class File(FileResource):
-
-    name = 'file'
-
-    def __init__(self, owner, path, url=None):
-        super(File, self).__init__(owner)
-        self.path = path
-        self.url = url
-
-    def __str__(self):
-        return '<{}\'s {} {}>'.format(self.owner, self.name, self.path or self.url)
-
-
-class PluginAsset(File):
-
-    name = 'plugin_asset'
 
     def __init__(self, owner, path):
-        super(PluginAsset, self).__init__(owner, os.path.join(owner.name, path))
+        super(File, self).__init__(owner)
+        self.path = path
 
-
+    def match(self, path):
-class Executable(FileResource):
+        return self.path == path
-
-    name = 'executable'
-
-    def __init__(self, owner, platform, filename):
-        super(Executable, self).__init__(owner)
-        self.platform = platform
-        self.filename = filename
 
     def __str__(self):
-        return '<{}\'s {} {}>'.format(self.owner, self.platform, self.filename)
+        return '<{}\'s {} {}>'.format(self.owner, self.kind, self.path)
 
-class ReventFile(FileResource):
 
-    name = 'revent'
+class Executable(Resource):
 
-    def __init__(self, owner, stage):
+    kind = 'executable'
+
+    def __init__(self, owner, abi, filename):
+        super(Executable, self).__init__(owner)
+        self.abi = abi
+        self.filename = filename
+
+    def match(self, path):
+        return self.filename == os.path.basename(path)
+
+    def __str__(self):
+        return '<{}\'s {} {}>'.format(self.owner, self.abi, self.filename)
+
+
+class ReventFile(Resource):
+
+    kind = 'revent'
+
+    def __init__(self, owner, stage, target):
         super(ReventFile, self).__init__(owner)
         self.stage = stage
+        self.target = target
+
+    def match(self, path):
+        filename = os.path.basename(path)
+        parts = filename.split('.')
+        if len(parts) > 2:
+            target, stage = parts[:2]
+            return target == self.target and stage == self.stage
+        else:
+            stage = parts[0]
+            return stage == self.stage
 
 
-class JarFile(FileResource):
+class JarFile(Resource):
 
-    name = 'jar'
+    kind = 'jar'
+
+    def match(self, path):
+        # An owner always  has at most one jar file, so
+        # always match
+        return True
 
 
-class ApkFile(FileResource):
+class ApkFile(Resource):
 
-    name = 'apk'
+    kind = 'apk'
 
-    def __init__(self, owner, version):
+    def __init__(self, owner, variant=None, version=None):
         super(ApkFile, self).__init__(owner)
+        self.variant = variant
         self.version = version
 
+    def match(self, path):
+        name_matches = True
+        version_matches = True
+        if self.version is not None:
+            version_matches = apk_version_matches(path, self.version)
+        if self.variant is not None:
+            name_matches = file_name_matches(path, self.variant)
+        return name_matches and version_matches:
+
+    def __str__(self):
+        text = '<{}\'s apk'.format(self.owner)
+        if self.variant:
+            text += ' {}'.format(self.variant)
+        if self.version:
+            text += ' {}'.format(self.version)
+        text += '>'
+        return text
+
 
 class ResourceGetter(Plugin):
     """
@@ -192,10 +180,6 @@ class ResourceGetter(Plugin):
 
     :name: Name that uniquely identifies this getter. Must be set by any
            concrete subclass.
-    :resource_type: Identifies resource type(s) that this getter can
-                    handle. This must be either a string (for a single type)
-                    or a list of strings for multiple resource types. This
-                    must be set by any concrete subclass.
     :priority: Priority with which this getter will be invoked. This should
                be one of the standard priorities specified in
                ``GetterPriority`` enumeration. If not set, this will default
@@ -205,74 +189,12 @@ class ResourceGetter(Plugin):
 
     name = None
     kind = 'resource_getter'
-    resource_type = None
-    priority = GetterPriority.environment
 
-    def __init__(self, resolver, **kwargs):
+    def register(self, resolver):
-        super(ResourceGetter, self).__init__(**kwargs)
-        self.resolver = resolver
-
-    def register(self):
-        """
-        Registers with a resource resolver. Concrete implementations must
-        override this to invoke ``self.resolver.register()`` method to register
-        ``self`` for specific resource types.
-
-        """
-        if self.resource_type is None:
-            message = 'No resource type specified for {}'
-            raise ValueError(message.format(self.name))
-        elif isinstance(self.resource_type, list):
-            for rt in self.resource_type:
-                self.resolver.register(self, rt, self.priority)
-        else:
-            self.resolver.register(self, self.resource_type, self.priority)
-
-    def unregister(self):
-        """Unregister from a resource resolver."""
-        if self.resource_type is None:
-            message = 'No resource type specified for {}'
-            raise ValueError(message.format(self.name))
-        elif isinstance(self.resource_type, list):
-            for rt in self.resource_type:
-                self.resolver.unregister(self, rt)
-        else:
-            self.resolver.unregister(self, self.resource_type)
-
-    def get(self, resource, **kwargs):
-        """
-        This will get invoked by the resolver when attempting to resolve a
-        resource, passing in the resource to be resolved as the first
-        parameter. Any additional parameters would be specific to a particular
-        resource type.
-
-        This method will only be invoked for resource types that the getter has
-        registered for.
-
-        :param resource: an instance of :class:`wlauto.core.resource.Resource`.
-
-        :returns: Implementations of this method must return either the
-                  discovered resource or ``None`` if the resource could not
-                  be discovered.
-
-        """
         raise NotImplementedError()
 
-    def delete(self, resource, *args, **kwargs):
+    def initialize(self):
-        """
+        pass
-        Delete the resource if it is discovered. All arguments are passed to a
-        call to``self.get()``. If that call returns a resource, it is deleted.
-
-        :returns: ``True`` if the specified resource has been discovered
-                  and deleted, and ``False`` otherwise.
-
-        """
-        discovered = self.get(resource, *args, **kwargs)
-        if discovered:
-            resource.delete(discovered)
-            return True
-        else:
-            return False
 
     def __str__(self):
         return '<ResourceGetter {}>'.format(self.name)
@@ -285,23 +207,31 @@ class ResourceResolver(object):
 
     """
 
-    def __init__(self, config):
+    def __init__(self, loader=pluginloader):
+        self.loader = loader
         self.logger = logging.getLogger('resolver')
-        self.getters = defaultdict(prioritylist)
+        self.getters = []
-        self.config = config
+        self.sources = prioritylist()
 
     def load(self):
-        """
+        for gettercls in self.loader.list_plugins('resource_getter'):
-        Discover getters under the specified source. The source could
+            self.logger.debug('Loading getter {}'.format(gettercls.name))
-        be either a python package/module or a path.
+            getter = self.loader.get_plugin(name=gettercls.name, 
+                                            kind="resource_getter")
+            log.indent()
+            try:
+                getter.initialize()
+                getter.register(self)
+            finally:
+                log.dedent()
+            self.getters.append(getter)
 
-        """
+    def register(self, source, priority=SourcePriority.local):
+        msg = 'Registering "{}" with priority "{}"'
+        self.logger.debug(msg.format(get_object_name(source), priority))
+        self.sources.add(source, priority)
 
-        for rescls in pluginloader.list_resource_getters():
+    def get(self, resource, strict=True):
-            getter = self.config.get_plugin(name=rescls.name, kind="resource_getter", resolver=self)
-            getter.register()
-
-    def get(self, resource, strict=True, *args, **kwargs):
         """
         Uses registered getters to attempt to discover a resource of the specified
         kind and matching the specified criteria. Returns path to the resource that
@@ -311,11 +241,13 @@ class ResourceResolver(object):
 
         """
         self.logger.debug('Resolving {}'.format(resource))
-        for getter in self.getters[resource.name]:
+        for source in self.sources:
-            self.logger.debug('Trying {}'.format(getter))
+            source_name = get_object_name(source)
-            result = getter.get(resource, *args, **kwargs)
+            self.logger.debug('Trying {}'.format(source_name))
+            result = source(resource)
             if result is not None:
-                self.logger.debug('Resource {} found using {}:'.format(resource, getter))
+                msg = 'Resource {} found using {}:'
+                self.logger.debug(msg.format(resource, source_name))
                 self.logger.debug('\t{}'.format(result))
                 return result
         if strict:
@@ -323,61 +255,19 @@ class ResourceResolver(object):
         self.logger.debug('Resource {} not found.'.format(resource))
         return None
 
-    def register(self, getter, kind, priority=0):
-        """
-        Register the specified resource getter as being able to discover a resource
-        of the specified kind with the specified priority.
 
-        This method would typically be invoked by a getter inside its __init__.
+def apk_version_matches(path, version):
-        The idea being that getters register themselves for resources they know
+    info = ApkInfo(path)
-        they can discover.
+    if info.version_name == version or info.version_code == version:
-
+        return True
-        *priorities*
+    return False
-
-        getters that are registered with the highest priority will be invoked first. If
-        multiple getters are registered under the same priority, they will be invoked
-        in the order they were registered (i.e. in the order they were discovered). This is
-        essentially non-deterministic.
-
-        Generally getters that are more likely to find a resource, or would find a
-        "better" version of the resource should register with higher (positive) priorities.
-        Fall-back getters that should only be invoked if a resource is not found by usual
-        means should register with lower (negative) priorities.
-
-        """
-        self.logger.debug('Registering {} for {} resources'.format(getter.name, kind))
-        self.getters[kind].add(getter, priority)
-
-    def unregister(self, getter, kind):
-        """
-        Unregister a getter that has been registered earlier.
-
-        """
-        self.logger.debug('Unregistering {}'.format(getter.name))
-        try:
-            self.getters[kind].remove(getter)
-        except ValueError:
-            raise ValueError('Resource getter {} is not installed.'.format(getter.name))
-
-# Utility functions
-
-def get_from_location_by_extension(resource, location, extension, version=None):
-    found_files = glob.glob(os.path.join(location, '*.{}'.format(extension)))
-    if version:
-        found_files = [ff for ff in found_files 
-                       if version.lower() in os.path.basename(ff).lower()]
-    if len(found_files) == 1:
-        return found_files[0]
-    elif not found_files:
-        return None
-    else:
-        raise ResourceError('More than one .{} found in {} for {}.'.format(extension,
-                                                                           location,
-                                                                           resource.owner.name))
 
 
-def _get_owner_path(resource):
+def file_name_matches(path, pattern):
-    if resource.owner is NO_ONE:
+    filename = os.path.basename(path)
-        return os.path.join(os.path.dirname(__base_filepath), 'common')
+    if pattern in filename:
-    else:
+        return True
-        return os.path.dirname(sys.modules[resource.owner.__module__].__file__)
+    if re.search(pattern, filename):
+        return True
+    return False
+

wa/framework/resource_getters.py

@@ -1,510 +0,0 @@
-#    Copyright 2013-2015 ARM Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-"""
-This module contains the standard set of resource getters used by Workload Automation.
-
-"""
-import os
-import sys
-import shutil
-import inspect
-import httplib
-import logging
-import json
-
-import requests
-
-from wa import Parameter, settings, __file__ as __base_filepath
-from wa.framework.resource import ResourceGetter, GetterPriority, NO_ONE 
-from wa.framework.exception import ResourceError
-from wa.utils.misc import (ensure_directory_exists as _d, 
-                           ensure_file_directory_exists as _f, sha256, urljoin)
-from wa.utils.types import boolean
-
-
-logging.getLogger("requests").setLevel(logging.WARNING)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-
-class PackageFileGetter(ResourceGetter):
-
-    name = 'package_file'
-    description = """
-    Looks for exactly one file with the specified plugin in the owner's directory. If a version
-    is specified on invocation of get, it will filter the discovered file based on that version.
-    Versions are treated as case-insensitive.
-    """
-
-    plugin = None
-
-    def register(self):
-        self.resolver.register(self, self.plugin, GetterPriority.package)
-
-    def get(self, resource, **kwargs):
-        resource_dir = os.path.dirname(sys.modules[resource.owner.__module__].__file__)
-        version = kwargs.get('version')
-        return get_from_location_by_plugin(resource, resource_dir, self.plugin, version)
-
-
-class EnvironmentFileGetter(ResourceGetter):
-
-    name = 'environment_file'
-    description = """Looks for exactly one file with the specified plugin in the owner's directory. If a version
-    is specified on invocation of get, it will filter the discovered file based on that version.
-    Versions are treated as case-insensitive."""
-
-    plugin = None
-
-    def register(self):
-        self.resolver.register(self, self.plugin, GetterPriority.environment)
-
-    def get(self, resource, **kwargs):
-        resource_dir = resource.owner.dependencies_directory
-
-        version = kwargs.get('version')
-        return get_from_location_by_plugin(resource, resource_dir, self.plugin, version)
-
-
-class ReventGetter(ResourceGetter):
-    """Implements logic for identifying revent files."""
-
-    def get_base_location(self, resource):
-        raise NotImplementedError()
-
-    def register(self):
-        self.resolver.register(self, 'revent', GetterPriority.package)
-
-    def get(self, resource, **kwargs):
-        filename = '.'.join([resource.owner.device.model, resource.stage, 'revent']).lower()
-        location = _d(os.path.join(self.get_base_location(resource), 'revent_files'))
-        for candidate in os.listdir(location):
-            if candidate.lower() == filename.lower():
-                return os.path.join(location, candidate)
-
-
-class PackageApkGetter(PackageFileGetter):
-    name = 'package_apk'
-    plugin = 'apk'
-
-
-class PackageJarGetter(PackageFileGetter):
-    name = 'package_jar'
-    plugin = 'jar'
-
-
-class PackageReventGetter(ReventGetter):
-
-    name = 'package_revent'
-
-    def get_base_location(self, resource):
-        return get_owner_path(resource)
-
-
-class EnvironmentApkGetter(EnvironmentFileGetter):
-    name = 'environment_apk'
-    plugin = 'apk'
-
-
-class EnvironmentJarGetter(EnvironmentFileGetter):
-    name = 'environment_jar'
-    plugin = 'jar'
-
-
-class EnvironmentReventGetter(ReventGetter):
-
-    name = 'enviroment_revent'
-
-    def get_base_location(self, resource):
-        return resource.owner.dependencies_directory
-
-
-class ExecutableGetter(ResourceGetter):
-
-    name = 'exe_getter'
-    resource_type = 'executable'
-    priority = GetterPriority.environment
-
-    def get(self, resource, **kwargs):
-        if settings.assets_repository:
-            path = os.path.join(settings.assets_repository, resource.platform, resource.filename)
-            if os.path.isfile(path):
-                return path
-
-
-class PackageExecutableGetter(ExecutableGetter):
-
-    name = 'package_exe_getter'
-    priority = GetterPriority.package
-
-    def get(self, resource, **kwargs):
-        path = os.path.join(get_owner_path(resource), 'bin', resource.platform, resource.filename)
-        if os.path.isfile(path):
-            return path
-
-
-class EnvironmentExecutableGetter(ExecutableGetter):
-
-    name = 'env_exe_getter'
-
-    def get(self, resource, **kwargs):
-        paths = [
-            os.path.join(resource.owner.dependencies_directory, 'bin',
-                         resource.platform, resource.filename),
-            os.path.join(settings.user_directory, 'bin',
-                         resource.platform, resource.filename),
-        ]
-        for path in paths:
-            if os.path.isfile(path):
-                return path
-
-
-class DependencyFileGetter(ResourceGetter):
-
-    name = 'filer'
-    description = """
-    Gets resources from the specified mount point. Copies them the local dependencies
-    directory, and returns the path to the local copy.
-
-    """
-    resource_type = 'file'
-    relative_path = ''  # May be overridden by subclasses.
-
-    priority = GetterPriority.remote
-
-    parameters = [
-        Parameter('mount_point', default='/', global_alias='remote_assets_path',
-                  description='Local mount point for the remote filer.'),
-    ]
-
-    def __init__(self, resolver, **kwargs):
-        super(DependencyFileGetter, self).__init__(resolver, **kwargs)
-
-    def get(self, resource, **kwargs):
-        force = kwargs.get('force')
-        remote_path = os.path.join(self.mount_point, self.relative_path, resource.path)
-        local_path = os.path.join(resource.owner.dependencies_directory, os.path.basename(resource.path))
-
-        if not os.path.isfile(local_path) or force:
-            if not os.path.isfile(remote_path):
-                return None
-            self.logger.debug('Copying {} to {}'.format(remote_path, local_path))
-            shutil.copy(remote_path, local_path)
-
-        return local_path
-
-
-class PackageCommonDependencyGetter(ResourceGetter):
-
-    name = 'packaged_common_dependency'
-    resource_type = 'file'
-    priority = GetterPriority.package - 1  # check after owner-specific locations
-
-    def get(self, resource, **kwargs):
-        path = os.path.join(settings.package_directory, 'common', resource.path)
-        if os.path.exists(path):
-            return path
-
-
-class EnvironmentCommonDependencyGetter(ResourceGetter):
-
-    name = 'environment_common_dependency'
-    resource_type = 'file'
-    priority = GetterPriority.environment - 1  # check after owner-specific locations
-
-    def get(self, resource, **kwargs):
-        path = os.path.join(settings.dependencies_directory,
-                            os.path.basename(resource.path))
-        if os.path.exists(path):
-            return path
-
-
-class PackageDependencyGetter(ResourceGetter):
-
-    name = 'packaged_dependency'
-    resource_type = 'file'
-    priority = GetterPriority.package
-
-    def get(self, resource, **kwargs):
-        owner_path = inspect.getfile(resource.owner.__class__)
-        path = os.path.join(os.path.dirname(owner_path), resource.path)
-        if os.path.exists(path):
-            return path
-
-
-class EnvironmentDependencyGetter(ResourceGetter):
-
-    name = 'environment_dependency'
-    resource_type = 'file'
-    priority = GetterPriority.environment
-
-    def get(self, resource, **kwargs):
-        path = os.path.join(resource.owner.dependencies_directory, os.path.basename(resource.path))
-        if os.path.exists(path):
-            return path
-
-
-class PluginAssetGetter(DependencyFileGetter):
-
-    name = 'plugin_asset'
-    resource_type = 'plugin_asset'
-
-
-class HttpGetter(ResourceGetter):
-
-    name = 'http_assets'
-    description = """
-    Downloads resources from a server based on an index fetched from the specified URL.
-
-    Given a URL, this will try to fetch ``<URL>/index.json``. The index file maps plugin
-    names to a list of corresponing asset descriptons. Each asset description continas a path
-    (relative to the base URL) of the resource and a SHA256 hash, so that this Getter can
-    verify whether the resource on the remote has changed.
-
-    For example, let's assume we want to get the APK file for workload "foo", and that
-    assets are hosted at ``http://example.com/assets``. This Getter will first try to
-    donwload ``http://example.com/assests/index.json``. The index file may contian
-    something like ::
-
-        {
-            "foo": [
-                {
-                    "path": "foo-app.apk",
-                    "sha256": "b14530bb47e04ed655ac5e80e69beaa61c2020450e18638f54384332dffebe86"
-                },
-                {
-                    "path": "subdir/some-other-asset.file",
-                    "sha256": "48d9050e9802246d820625717b72f1c2ba431904b8484ca39befd68d1dbedfff"
-                }
-            ]
-        }
-
-    This Getter will look through the list of assets for "foo" (in this case, two) check
-    the paths until it finds one matching the resource (in this case, "foo-app.apk").
-    Finally, it will try to dowload that file relative to the base URL and plugin name
-    (in this case, "http://example.com/assets/foo/foo-app.apk"). The downloaded version
-    will be cached locally, so that in the future, the getter will check the SHA256 hash
-    of the local file against the one advertised inside index.json, and provided that hasn't
-    changed, it won't try to download the file again.
-
-    """
-    priority = GetterPriority.remote
-    resource_type = ['apk', 'file', 'jar', 'revent']
-
-    parameters = [
-        Parameter('url', global_alias='remote_assets_url',
-                  description="""URL of the index file for assets on an HTTP server."""),
-        Parameter('username',
-                  description="""User name for authenticating with assets URL"""),
-        Parameter('password',
-                  description="""Password for authenticationg with assets URL"""),
-        Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets',
-                  description="""If ``True``, will always attempt to fetch assets from the remote, even if
-                                 a local cached copy is available."""),
-        Parameter('chunk_size', kind=int, default=1024,
-                  description="""Chunk size for streaming large assets."""),
-    ]
-
-    def __init__(self, resolver, **kwargs):
-        super(HttpGetter, self).__init__(resolver, **kwargs)
-        self.index = None
-
-    def get(self, resource, **kwargs):
-        if not resource.owner:
-            return  # TODO: add support for unowned resources
-        if not self.index:
-            self.index = self.fetch_index()
-        asset = self.resolve_resource(resource)
-        if not asset:
-            return
-        return self.download_asset(asset, resource.owner.name)
-
-    def fetch_index(self):
-        if not self.url:
-            return {}
-        index_url = urljoin(self.url, 'index.json')
-        response = self.geturl(index_url)
-        if response.status_code != httplib.OK:
-            message = 'Could not fetch "{}"; recieved "{} {}"'
-            self.logger.error(message.format(index_url, response.status_code, response.reason))
-            return {}
-        return json.loads(response.content)
-
-    def download_asset(self, asset, owner_name):
-        url = urljoin(self.url, owner_name, asset['path'])
-        local_path = _f(os.path.join(settings.dependencies_directory, '__remote',
-                                     owner_name, asset['path'].replace('/', os.sep)))
-        if os.path.isfile(local_path) and not self.always_fetch:
-            local_sha = sha256(local_path)
-            if local_sha == asset['sha256']:
-                self.logger.debug('Local SHA256 matches; not re-downloading')
-                return local_path
-        self.logger.debug('Downloading {}'.format(url))
-        response = self.geturl(url, stream=True)
-        if response.status_code != httplib.OK:
-            message = 'Could not download asset "{}"; recieved "{} {}"'
-            self.logger.warning(message.format(url, response.status_code, response.reason))
-            return
-        with open(local_path, 'wb') as wfh:
-            for chunk in response.iter_content(chunk_size=self.chunk_size):
-                wfh.write(chunk)
-        return local_path
-
-    def geturl(self, url, stream=False):
-        if self.username:
-            auth = (self.username, self.password)
-        else:
-            auth = None
-        return requests.get(url, auth=auth, stream=stream)
-
-    def resolve_resource(self, resource):
-        assets = self.index.get(resource.owner.name, {})
-        if not assets:
-            return {}
-        if resource.name in ['apk', 'jar']:
-            paths = [a['path'] for a in assets]
-            version = getattr(resource, 'version', None)
-            found = get_from_list_by_plugin(resource, paths, resource.name, version)
-            if found:
-                for a in assets:
-                    if a['path'] == found:
-                        return a
-        elif resource.name == 'revent':
-            filename = '.'.join([resource.owner.device.name, resource.stage, 'revent']).lower()
-            for asset in assets:
-                pathname = os.path.basename(asset['path']).lower()
-                if pathname == filename:
-                    return asset
-        else:  # file
-            for asset in assets:
-                if asset['path'].lower() == resource.path.lower():
-                    return asset
-
-
-class RemoteFilerGetter(ResourceGetter):
-
-    name = 'filer_assets'
-    description = """
-    Finds resources on a (locally mounted) remote filer and caches them locally.
-
-    This assumes that the filer is mounted on the local machine (e.g. as a samba share).
-
-    """
-    priority = GetterPriority.remote
-    resource_type = ['apk', 'file', 'jar', 'revent']
-
-    parameters = [
-        Parameter('remote_path', global_alias='remote_assets_path', default='',
-                  description="""Path, on the local system, where the assets are located."""),
-        Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets',
-                  description="""If ``True``, will always attempt to fetch assets from the remote, even if
-                                 a local cached copy is available."""),
-    ]
-
-    def get(self, resource, **kwargs):
-        version = kwargs.get('version')
-        if resource.owner:
-            remote_path = os.path.join(self.remote_path, resource.owner.name)
-            local_path = os.path.join(settings.user_directory, '__filer', resource.owner.dependencies_directory)
-            return self.try_get_resource(resource, version, remote_path, local_path)
-        else:
-            result = None
-            for entry in os.listdir(remote_path):
-                remote_path = os.path.join(self.remote_path, entry)
-                local_path = os.path.join(settings.user_directory, '__filer', settings.dependencies_directory, entry)
-                result = self.try_get_resource(resource, version, remote_path, local_path)
-                if result:
-                    break
-            return result
-
-    def try_get_resource(self, resource, version, remote_path, local_path):
-        if not self.always_fetch:
-            result = self.get_from(resource, version, local_path)
-            if result:
-                return result
-        if remote_path:
-            # Didn't find it cached locally; now check the remoted
-            result = self.get_from(resource, version, remote_path)
-            if not result:
-                return result
-        else:  # remote path is not set
-            return None
-        # Found it remotely, cache locally, then return it
-        local_full_path = os.path.join(_d(local_path), os.path.basename(result))
-        self.logger.debug('cp {} {}'.format(result, local_full_path))
-        shutil.copy(result, local_full_path)
-        return local_full_path
-
-    def get_from(self, resource, version, location):  # pylint: disable=no-self-use
-        if resource.name in ['apk', 'jar']:
-            return get_from_location_by_plugin(resource, location, resource.name, version)
-        elif resource.name == 'file':
-            filepath = os.path.join(location, resource.path)
-            if os.path.exists(filepath):
-                return filepath
-        elif resource.name == 'revent':
-            filename = '.'.join([resource.owner.device.model, resource.stage, 'revent']).lower()
-            alternate_location = os.path.join(location, 'revent_files')
-            # There tends to be some confusion as to where revent files should
-            # be placed. This looks both in the plugin's directory, and in
-            # 'revent_files' subdirectory under it, if it exists.
-            if os.path.isdir(alternate_location):
-                for candidate in os.listdir(alternate_location):
-                    if candidate.lower() == filename.lower():
-                        return os.path.join(alternate_location, candidate)
-            if os.path.isdir(location):
-                for candidate in os.listdir(location):
-                    if candidate.lower() == filename.lower():
-                        return os.path.join(location, candidate)
-        else:
-            raise ValueError('Unexpected resource type: {}'.format(resource.name))
-
-
-# Utility functions
-
-def get_from_location_by_plugin(resource, location, plugin, version=None):
-    try:
-        found_files = [os.path.join(location, f) for f in os.listdir(location)]
-    except OSError:
-        return None
-    try:
-        return get_from_list_by_plugin(resource, found_files, plugin, version)
-    except ResourceError:
-        raise ResourceError('More than one .{} found in {} for {}.'.format(plugin,
-                                                                           location,
-                                                                           resource.owner.name))
-
-
-def get_from_list_by_plugin(resource, filelist, plugin, version=None):
-    filelist = [ff for ff in filelist
-                if os.path.splitext(ff)[1].lower().endswith(plugin)]
-    if version:
-        filelist = [ff for ff in filelist if version.lower() in os.path.basename(ff).lower()]
-    if len(filelist) == 1:
-        return filelist[0]
-    elif not filelist:
-        return None
-    else:
-        raise ResourceError('More than one .{} found in {} for {}.'.format(plugin,
-                                                                           filelist,
-                                                                           resource.owner.name))
-
-
-def get_owner_path(resource):
-    if resource.owner is NO_ONE:
-        return os.path.join(os.path.dirname(__base_filepath), 'common')
-    else:
-        return os.path.dirname(sys.modules[resource.owner.__module__].__file__)

wa/utils/misc.py

@@ -583,3 +583,17 @@ def merge_dicts_simple(base, other):
 def touch(path):
     with open(path, 'w'):
         pass
+
+
+def get_object_name(obj):
+    if hasattr(obj, 'name'):
+        return obj.name
+    elif hasattr(obj,'func_name'):
+        return obj.func_name
+    elif hasattr(obj, 'im_func'):
+        return '{}.{}'.format(obj.im_class.__name__, obj.im_func.func_name)
+    elif hasattr(obj, '__name__'):
+        return obj.__name__
+    elif hasattr(obj, '__class__'):
+        return obj.__class__.__name__
+    return None