workload-automation/wa/framework/resource.py

#    Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import sys
import glob
import shutil
import inspect
import logging
from collections import defaultdict

from wa.framework import pluginloader
from wa.framework.plugin import Plugin, Parameter
from wa.framework.exception import ResourceError
from wa.framework.configuration import settings
from wa.utils.misc import ensure_directory_exists as _d
from wa.utils.types import boolean
from wa.utils.types import prioritylist


class GetterPriority(object):
    """
    Enumerates standard ResourceGetter priorities. In general, getters should
    register under one of these, rather than specifying other priority values.


    :cached: The cached version of the resource. Look here first. This
    priority also implies
             that the resource at this location is a "cache" and is not
             the only version of the resource, so it may be cleared without
             losing access to the resource.
    :preferred: Take this resource in favour of the environment resource.
    :environment: Found somewhere under ~/.workload_automation/ or equivalent,
                  or from environment variables, external configuration
                  files, etc.  These will override resource supplied with
                  the package.
    :external_package: Resource provided by another package.  :package:
                       Resource provided with the package.  :remote:
                       Resource will be downloaded from a remote location
                       (such as an HTTP server or a samba share). Try this
                       only if no other getter was successful.

    """
    cached = 20
    preferred = 10
    environment = 0
    external_package = -5
    package = -10
    remote = -20


class Resource(object):
    """
    Represents a resource that needs to be resolved. This can be pretty much
    anything: a file, environment variable, a Python object, etc. The only
    thing a resource *has* to have is an owner (which would normally be the
    Workload/Instrument/Device/etc object that needs the resource). In
    addition, a resource have any number of attributes to identify, but all of
    them are resource type specific.

    """

    name = None

    def __init__(self, owner):
        self.owner = owner

    def delete(self, instance):
        """
        Delete an instance of this resource type. This must be implemented
        by the concrete subclasses based on what the resource looks like,
        e.g. deleting a file or a directory tree, or removing an entry from
        a database.

        :note: Implementation should *not* contain any logic for deciding
               whether or not a resource should be deleted, only the actual
               deletion. The assumption is that if this method is invoked,
               then the decision has already been made.

        """
        raise NotImplementedError()

    def __str__(self):
        return '<{}\'s {}>'.format(self.owner, self.name)


class ResourceGetter(Plugin):
    """
    Base class for implementing resolvers. Defines resolver
    interface. Resolvers are responsible for discovering resources (such as
    particular kinds of files) they know about based on the parameters that are
    passed to them. Each resolver also has a dict of attributes that describe
    it's operation, and may be used to determine which get invoked.  There is
    no pre-defined set of attributes and resolvers may define their own.

    Class attributes:

    :name: Name that uniquely identifies this getter. Must be set by any
           concrete subclass.
    :resource_type: Identifies resource type(s) that this getter can
                    handle. This must be either a string (for a single type)
                    or a list of strings for multiple resource types. This
                    must be set by any concrete subclass.
    :priority: Priority with which this getter will be invoked. This should
               be one of the standard priorities specified in
               ``GetterPriority`` enumeration. If not set, this will default
               to ``GetterPriority.environment``.

    """

    name = None
    kind = 'resource_getter'
    resource_type = None
    priority = GetterPriority.environment

    def __init__(self, resolver, **kwargs):
        super(ResourceGetter, self).__init__(**kwargs)
        self.resolver = resolver

    def register(self):
        """
        Registers with a resource resolver. Concrete implementations must
        override this to invoke ``self.resolver.register()`` method to register
        ``self`` for specific resource types.

        """
        if self.resource_type is None:
            message = 'No resource type specified for {}'
            raise ValueError(message.format(self.name))
        elif isinstance(self.resource_type, list):
            for rt in self.resource_type:
                self.resolver.register(self, rt, self.priority)
        else:
            self.resolver.register(self, self.resource_type, self.priority)

    def unregister(self):
        """Unregister from a resource resolver."""
        if self.resource_type is None:
            message = 'No resource type specified for {}'
            raise ValueError(message.format(self.name))
        elif isinstance(self.resource_type, list):
            for rt in self.resource_type:
                self.resolver.unregister(self, rt)
        else:
            self.resolver.unregister(self, self.resource_type)

    def get(self, resource, **kwargs):
        """
        This will get invoked by the resolver when attempting to resolve a
        resource, passing in the resource to be resolved as the first
        parameter. Any additional parameters would be specific to a particular
        resource type.

        This method will only be invoked for resource types that the getter has
        registered for.

        :param resource: an instance of :class:`wlauto.core.resource.Resource`.

        :returns: Implementations of this method must return either the
                  discovered resource or ``None`` if the resource could not
                  be discovered.

        """
        raise NotImplementedError()

    def delete(self, resource, *args, **kwargs):
        """
        Delete the resource if it is discovered. All arguments are passed to a
        call to``self.get()``. If that call returns a resource, it is deleted.

        :returns: ``True`` if the specified resource has been discovered
                  and deleted, and ``False`` otherwise.

        """
        discovered = self.get(resource, *args, **kwargs)
        if discovered:
            resource.delete(discovered)
            return True
        else:
            return False

    def __str__(self):
        return '<ResourceGetter {}>'.format(self.name)


class ResourceResolver(object):
    """
    Discovers and registers getters, and then handles requests for
    resources using registered getters.

    """

    def __init__(self):
        self.logger = logging.getLogger('resolver')
        self.getters = defaultdict(prioritylist)

    def load(self, loader=pluginloader):
        """
        Discover getters under the specified source. The source could
        be either a python package/module or a path.

        """
        for rescls in loader.list_resource_getters():
            getter = loader.get_resource_getter(rescls.name, resolver=self)
            getter.register()

    def get(self, resource, strict=True, *args, **kwargs):
        """
        Uses registered getters to attempt to discover a resource of the specified
        kind and matching the specified criteria. Returns path to the resource that
        has been discovered. If a resource has not been discovered, this will raise
        a ``ResourceError`` or, if ``strict`` has been set to ``False``, will return
        ``None``.

        """
        self.logger.debug('Resolving {}'.format(resource))
        for getter in self.getters[resource.name]:
            self.logger.debug('Trying {}'.format(getter))
            result = getter.get(resource, *args, **kwargs)
            if result is not None:
                self.logger.debug('Resource {} found using {}:'.format(resource, getter))
                self.logger.debug('\t{}'.format(result))
                return result
        if strict:
            raise ResourceError('{} could not be found'.format(resource))
        self.logger.debug('Resource {} not found.'.format(resource))
        return None

    def register(self, getter, kind, priority=0):
        """
        Register the specified resource getter as being able to discover a resource
        of the specified kind with the specified priority.

        This method would typically be invoked by a getter inside its __init__.
        The idea being that getters register themselves for resources they know
        they can discover.

        *priorities*

        getters that are registered with the highest priority will be invoked first. If
        multiple getters are registered under the same priority, they will be invoked
        in the order they were registered (i.e. in the order they were discovered). This is
        essentially non-deterministic.

        Generally getters that are more likely to find a resource, or would find a
        "better" version of the resource should register with higher (positive) priorities.
        Fall-back getters that should only be invoked if a resource is not found by usual
        means should register with lower (negative) priorities.

        """
        self.logger.debug('Registering {}'.format(getter.name))
        self.getters[kind].add(getter, priority)

    def unregister(self, getter, kind):
        """
        Unregister a getter that has been registered earlier.

        """
        self.logger.debug('Unregistering {}'.format(getter.name))
        try:
            self.getters[kind].remove(getter)
        except ValueError:
            raise ValueError('Resource getter {} is not installed.'.format(getter.name))


class __NullOwner(object):
    """Represents an owner for a resource not owned by anyone."""

    name = 'noone'
    dependencies_directory = settings.dependencies_directory

    def __getattr__(self, name):
        return None

    def __str__(self):
        return 'no-one'

    __repr__ = __str__


NO_ONE = __NullOwner()


class FileResource(Resource):
    """
    Base class for all resources that are a regular file in the
    file system.

    """

    def delete(self, instance):
        os.remove(instance)


class File(FileResource):

    name = 'file'

    def __init__(self, owner, path, url=None):
        super(File, self).__init__(owner)
        self.path = path
        self.url = url

    def __str__(self):
        return '<{}\'s {} {}>'.format(self.owner, self.name, self.path or self.url)


class ExtensionAsset(File):

    name = 'extension_asset'

    def __init__(self, owner, path):
        super(ExtensionAsset, self).__init__(
            owner, os.path.join(owner.name, path))


class Executable(FileResource):

    name = 'executable'

    def __init__(self, owner, platform, filename):
        super(Executable, self).__init__(owner)
        self.platform = platform
        self.filename = filename

    def __str__(self):
        return '<{}\'s {} {}>'.format(self.owner, self.platform, self.filename)


class ReventFile(FileResource):

    name = 'revent'

    def __init__(self, owner, stage):
        super(ReventFile, self).__init__(owner)
        self.stage = stage


class JarFile(FileResource):

    name = 'jar'


class ApkFile(FileResource):

    name = 'apk'


class PackageFileGetter(ResourceGetter):

    name = 'package_file'
    description = """
    Looks for exactly one file with the specified extension in the owner's
    directory. If a version is specified on invocation of get, it will filter
    the discovered file based on that version.  Versions are treated as
    case-insensitive.
    """

    extension = None

    def register(self):
        self.resolver.register(self, self.extension, GetterPriority.package)

    def get(self, resource, **kwargs):
        resource_dir = os.path.dirname(
            sys.modules[resource.owner.__module__].__file__)
        version = kwargs.get('version')
        return get_from_location_by_extension(resource, resource_dir, self.extension, version)


class EnvironmentFileGetter(ResourceGetter):

    name = 'environment_file'
    description = """
    Looks for exactly one file with the specified extension in the owner's
    directory. If a version is specified on invocation of get, it will filter
    the discovered file based on that version.  Versions are treated as
    case-insensitive.
    """

    extension = None

    def register(self):
        self.resolver.register(self, self.extension,
                               GetterPriority.environment)

    def get(self, resource, **kwargs):
        resource_dir = resource.owner.dependencies_directory
        version = kwargs.get('version')
        return get_from_location_by_extension(resource, resource_dir, self.extension, version)


class ReventGetter(ResourceGetter):
    """Implements logic for identifying revent files."""

    def get_base_location(self, resource):
        raise NotImplementedError()

    def register(self):
        self.resolver.register(self, 'revent', GetterPriority.package)

    def get(self, resource, **kwargs):
        filename = '.'.join([resource.owner.device.name,
                             resource.stage, 'revent']).lower()
        location = _d(os.path.join(
            self.get_base_location(resource), 'revent_files'))
        for candidate in os.listdir(location):
            if candidate.lower() == filename.lower():
                return os.path.join(location, candidate)


class PackageApkGetter(PackageFileGetter):
    name = 'package_apk'
    extension = 'apk'


class PackageJarGetter(PackageFileGetter):
    name = 'package_jar'
    extension = 'jar'


class PackageReventGetter(ReventGetter):

    name = 'package_revent'

    def get_base_location(self, resource):
        return _get_owner_path(resource)


class EnvironmentApkGetter(EnvironmentFileGetter):
    name = 'environment_apk'
    extension = 'apk'


class EnvironmentJarGetter(EnvironmentFileGetter):
    name = 'environment_jar'
    extension = 'jar'


class EnvironmentReventGetter(ReventGetter):

    name = 'enviroment_revent'

    def get_base_location(self, resource):
        return resource.owner.dependencies_directory


class ExecutableGetter(ResourceGetter):

    name = 'exe_getter'
    resource_type = 'executable'
    priority = GetterPriority.environment

    def get(self, resource, **kwargs):
        if settings.binaries_repository:
            path = os.path.join(settings.binaries_repository,
                                resource.platform, resource.filename)
            if os.path.isfile(path):
                return path


class PackageExecutableGetter(ExecutableGetter):

    name = 'package_exe_getter'
    priority = GetterPriority.package

    def get(self, resource, **kwargs):
        path = os.path.join(_get_owner_path(resource), 'bin',
                            resource.platform, resource.filename)
        if os.path.isfile(path):
            return path


class EnvironmentExecutableGetter(ExecutableGetter):

    name = 'env_exe_getter'

    def get(self, resource, **kwargs):
        paths = [
            os.path.join(resource.owner.dependencies_directory, 'bin',
                         resource.platform, resource.filename),
            os.path.join(settings.environment_root, 'bin',
                         resource.platform, resource.filename),
        ]
        for path in paths:
            if os.path.isfile(path):
                return path


class DependencyFileGetter(ResourceGetter):

    name = 'filer'
    description = """
    Gets resources from the specified mount point. Copies them the local dependencies
    directory, and returns the path to the local copy.

    """
    resource_type = 'file'
    relative_path = ''  # May be overridden by subclasses.

    default_mount_point = '/'
    priority = GetterPriority.remote

    parameters = [
        Parameter('mount_point', default='/', global_alias='filer_mount_point',
                  description='Local mount point for the remote filer.'),
    ]

    def __init__(self, resolver, **kwargs):
        super(DependencyFileGetter, self).__init__(resolver, **kwargs)
        self.mount_point = settings.filer_mount_point or self.default_mount_point

    def get(self, resource, **kwargs):
        force = kwargs.get('force')
        remote_path = os.path.join(
            self.mount_point, self.relative_path, resource.path)
        local_path = os.path.join(
            resource.owner.dependencies_directory, os.path.basename(resource.path))

        if not os.path.isfile(local_path) or force:
            if not os.path.isfile(remote_path):
                return None
            self.logger.debug('Copying {} to {}'.format(
                remote_path, local_path))
            shutil.copy(remote_path, local_path)

        return local_path


class PackageCommonDependencyGetter(ResourceGetter):

    name = 'packaged_common_dependency'
    resource_type = 'file'
    priority = GetterPriority.package - 1  # check after owner-specific locations

    def get(self, resource, **kwargs):
        path = os.path.join(settings.package_directory,
                            'common', resource.path)
        if os.path.exists(path):
            return path


class EnvironmentCommonDependencyGetter(ResourceGetter):

    name = 'environment_common_dependency'
    resource_type = 'file'
    # check after owner-specific locations
    priority = GetterPriority.environment - 1

    def get(self, resource, **kwargs):
        path = os.path.join(settings.dependencies_directory,
                            os.path.basename(resource.path))
        if os.path.exists(path):
            return path


class PackageDependencyGetter(ResourceGetter):

    name = 'packaged_dependency'
    resource_type = 'file'
    priority = GetterPriority.package

    def get(self, resource, **kwargs):
        owner_path = inspect.getfile(resource.owner.__class__)
        path = os.path.join(os.path.dirname(owner_path), resource.path)
        if os.path.exists(path):
            return path


class EnvironmentDependencyGetter(ResourceGetter):

    name = 'environment_dependency'
    resource_type = 'file'
    priority = GetterPriority.environment

    def get(self, resource, **kwargs):
        path = os.path.join(resource.owner.dependencies_directory,
                            os.path.basename(resource.path))
        if os.path.exists(path):
            return path


class ExtensionAssetGetter(DependencyFileGetter):

    name = 'extension_asset'
    resource_type = 'extension_asset'
    relative_path = 'workload_automation/assets'


class RemoteFilerGetter(ResourceGetter):

    name = 'filer_assets'
    description = """
    Finds resources on a (locally mounted) remote filer and caches them locally.

    This assumes that the filer is mounted on the local machine (e.g. as a samba share).

    """
    priority = GetterPriority.remote
    resource_type = ['apk', 'file', 'jar', 'revent']

    parameters = [
        Parameter('remote_path', global_alias='remote_assets_path', default='',
                  description="""
                  Path, on the local system, where the assets are located.
                  """),
        Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets',
                  description="""
                  If ``True``, will always attempt to fetch assets from the
                  remote, even if a local cached copy is available.
                  """),
    ]

    def get(self, resource, **kwargs):
        version = kwargs.get('version')
        if resource.owner:
            remote_path = os.path.join(self.remote_path, resource.owner.name)
            local_path = os.path.join(
                settings.environment_root, resource.owner.dependencies_directory)
            return self.try_get_resource(resource, version, remote_path, local_path)
        else:
            result = None
            for entry in os.listdir(remote_path):
                remote_path = os.path.join(self.remote_path, entry)
                local_path = os.path.join(
                    settings.environment_root, settings.dependencies_directory, entry)
                result = self.try_get_resource(
                    resource, version, remote_path, local_path)
                if result:
                    break
            return result

    def try_get_resource(self, resource, version, remote_path, local_path):
        if not self.always_fetch:
            result = self.get_from(resource, version, local_path)
            if result:
                return result
        if remote_path:
            # Didn't find it cached locally; now check the remoted
            result = self.get_from(resource, version, remote_path)
            if not result:
                return result
        else:  # remote path is not set
            return None
        # Found it remotely, cache locally, then return it
        local_full_path = os.path.join(
            _d(local_path), os.path.basename(result))
        self.logger.debug('cp {} {}'.format(result, local_full_path))
        shutil.copy(result, local_full_path)
        return local_full_path

    def get_from(self, resource, version, location):  # pylint: disable=no-self-use
        if resource.name in ['apk', 'jar']:
            return get_from_location_by_extension(resource, location, resource.name, version)
        elif resource.name == 'file':
            filepath = os.path.join(location, resource.path)
            if os.path.exists(filepath):
                return filepath
        elif resource.name == 'revent':
            filename = '.'.join(
                [resource.owner.device.name, resource.stage, 'revent']).lower()
            alternate_location = os.path.join(location, 'revent_files')
            # There tends to be some confusion as to where revent files should
            # be placed. This looks both in the extension's directory, and in
            # 'revent_files' subdirectory under it, if it exists.
            if os.path.isdir(alternate_location):
                for candidate in os.listdir(alternate_location):
                    if candidate.lower() == filename.lower():
                        return os.path.join(alternate_location, candidate)
            if os.path.isdir(location):
                for candidate in os.listdir(location):
                    if candidate.lower() == filename.lower():
                        return os.path.join(location, candidate)
        else:
            message = 'Unexpected resource type: {}'.format(resource.name)
            raise ValueError(message)


# Utility functions

def get_from_location_by_extension(resource, location, extension, version=None):
    found_files = glob.glob(os.path.join(location, '*.{}'.format(extension)))
    if version:
        found_files = [ff for ff in found_files 
                       if version.lower() in os.path.basename(ff).lower()]
    if len(found_files) == 1:
        return found_files[0]
    elif not found_files:
        return None
    else:
        raise ResourceError('More than one .{} found in {} for {}.'.format(extension,
                                                                           location,
                                                                           resource.owner.name))


def _get_owner_path(resource):
    if resource.owner is NO_ONE:
        return os.path.join(os.path.dirname(__base_filepath), 'common')
    else:
        return os.path.dirname(sys.modules[resource.owner.__module__].__file__)