Fixing things up to a point where "list" and "show" commands work.

setup.py

@@ -78,7 +78,7 @@ params = dict(
         'pyYAML',  # YAML-formatted agenda parsing
         'requests',  # Fetch assets over HTTP
         'devlib',  # Interacting with devices
-        'louie'  # Handles signal callbacks
+        'louie'  # callbacks dispatch
     ],
     extras_require={
         'other': ['jinja2', 'pandas>=0.13.1'],

wlauto/commands/run.py

@@ -22,9 +22,9 @@ import wlauto
 from wlauto import Command, settings
 from wlauto.core.execution import Executor
 from wlauto.utils.log import add_log_file
-from wlauto.core.configuration import RunConfiguration, WAConfiguration
+from wlauto.core.configuration import RunConfiguration
 from wlauto.core import pluginloader
-from wlauto.core.configuration_parsers import Agenda, ConfigFile, EnvrironmentVars, CommandLineArgs
+from wlauto.core.configuration.parsers import AgendaParser, ConfigParser, CommandLineArgsParser
 
 
 class RunCommand(Command):
@@ -74,7 +74,7 @@ class RunCommand(Command):
 
         # STAGE 1: Gather configuratation
 
-        env = EnvrironmentVars()
+        env = EnvironmentVars()
         args = CommandLineArgs(args)
 
         # STAGE 2.1a: Early WAConfiguration, required to find config files

wlauto/core/command.py

@@ -21,7 +21,7 @@ from wlauto.core.version import get_wa_version
 
 
 def init_argument_parser(parser):
-    parser.add_argument('-c', '--config', help='specify an additional config.py', action='append')
+    parser.add_argument('-c', '--config', help='specify an additional config.py', action='append', default=[])
     parser.add_argument('-v', '--verbose', action='count',
                         help='The scripts will produce verbose output.')
     parser.add_argument('--version', action='version', version='%(prog)s {}'.format(get_wa_version()))

wlauto/core/configuration/__init__.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
 from wlauto.core.configuration.configuration import (settings,
-                                                     WAConfiguration,
                                                      RunConfiguration,
                                                      JobGenerator,
                                                      ConfigurationPoint)

wlauto/core/configuration/configuration.py

@@ -492,36 +492,11 @@ class CpuFreqParameters(object):
 class Configuration(object):
 
     config_points = []
-    name = ""
+    name = ''
+
     # The below line must be added to all subclasses
     configuration = {cp.name: cp for cp in config_points}
 
-    def __init__(self):
-        # Load default values for configuration points
-        for confpoint in self.configuration.itervalues():
-            confpoint.set_value(self, check_mandatory=False)
-
-    def set(self, name, value, check_mandatory=True):
-        if name not in self.configuration:
-            raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
-        self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
-
-    def update_config(self, values, check_mandatory=True):
-        for k, v in values.iteritems():
-            self.set(k, v, check_mandatory=check_mandatory)
-
-    def validate(self):
-        for cfg_point in self.configuration.itervalues():
-            cfg_point.validate(self)
-
-    def to_pod(self):
-        pod = {}
-        for cfg_point_name in self.configuration.iterkeys():
-            value = getattr(self, cfg_point_name, None)
-            if value is not None:
-                pod[cfg_point_name] = value
-        return pod
-
     @classmethod
     # pylint: disable=unused-argument
     def from_pod(cls, pod, plugin_cache):
@@ -535,11 +510,46 @@ class Configuration(object):
         instance.validate()
         return instance
 
+    def __init__(self):
+        for confpoint in self.config_points:
+            confpoint.set_value(self, check_mandatory=False)
+
+    def set(self, name, value, check_mandatory=True):
+        if name not in self.configuration:
+            raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
+        self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
+
+    def update_config(self, values, check_mandatory=True):
+        for k, v in values.iteritems():
+            self.set(k, v, check_mandatory=check_mandatory)
+
+    def validate(self):
+        for cfg_point in self.config_points:
+            cfg_point.validate(self)
+
+    def to_pod(self):
+        pod = {}
+        for cfg_point_name in self.configuration.iterkeys():
+            value = getattr(self, cfg_point_name, None)
+            if value is not None:
+                pod[cfg_point_name] = value
+        return pod
+
 
 # This configuration for the core WA framework
 class WAConfiguration(Configuration):
 
     name = "WA Configuration"
+
+    plugin_packages = [
+        'wlauto.commands',
+        'wlauto.workloads',
+        'wlauto.instrumentation',
+        'wlauto.result_processors',
+        'wlauto.managers',
+        'wlauto.resource_getters',
+    ]
+
     config_points = [
         ConfigurationPoint(
             'user_directory',
@@ -550,48 +560,6 @@ class WAConfiguration(Configuration):
             kind=str,
             default=os.path.join(os.path.expanduser('~'), '.workload_automation'),
         ),
-        ConfigurationPoint(
-            'plugin_packages',
-            kind=list_of_strings,
-            default=[
-                'wlauto.commands',
-                'wlauto.workloads',
-                'wlauto.instrumentation',
-                'wlauto.result_processors',
-                'wlauto.managers',
-                'wlauto.resource_getters',
-            ],
-            description="""
-            List of packages that will be scanned for WA plugins.
-            """,
-        ),
-        ConfigurationPoint(
-            'plugin_paths',
-            kind=list_of_strings,
-            default=[
-                'workloads',
-                'instruments',
-                'targets',
-                'processors',
-
-                # Legacy
-                'managers',
-                'result_processors',
-            ],
-            description="""
-            List of paths that will be scanned for WA plugins.
-            """,
-            merge=True
-        ),
-        ConfigurationPoint(
-            'plugin_ignore_paths',
-            kind=list_of_strings,
-            default=[],
-            description="""
-            List of (sub)paths that will be ignored when scanning
-            ``plugin_paths`` for WA plugins.
-            """,
-        ),
         ConfigurationPoint(
             'assets_repository',
             description="""
@@ -623,7 +591,7 @@ class WAConfiguration(Configuration):
             Verbosity of console output.
             """,
         ),
-        ConfigurationPoint(  # TODO: Needs some format for dates ect/ comes from cfg
+        ConfigurationPoint(  # TODO: Needs some format for dates etc/ comes from cfg
             'default_output_directory',
             default="wa_output",
             description="""
@@ -636,7 +604,19 @@ class WAConfiguration(Configuration):
 
     @property
     def dependencies_directory(self):
-        return "{}/dependencies/".format(self.user_directory)
+        return os.path.join(self.user_directory, 'dependencies')
+
+    @property
+    def plugins_directory(self):
+        return os.path.join(self.user_directory, 'plugins')
+
+
+    def __init__(self, environ):
+        super(WAConfiguration, self).__init__()
+        user_directory = environ.pop('WA_USER_DIRECTORY', '')
+        if user_directory:
+            self.set('user_directory', user_directory)
+
 
 
 # This is generic top-level configuration for WA runs.
@@ -1029,4 +1009,4 @@ class JobGenerator(object):
 
                 yield job_spec
 
-settings = WAConfiguration()
+settings = WAConfiguration(os.environ)

wlauto/core/configuration/parsers.py

@@ -283,23 +283,11 @@ class AgendaParser(object):
             raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
 
 
-class EnvironmentVarsParser(object):
-    def __init__(self, wa_config, environ):
-        user_directory = environ.pop('WA_USER_DIRECTORY', '')
-        if user_directory:
-            wa_config.set('user_directory', user_directory)
-        plugin_paths = environ.pop('WA_PLUGIN_PATHS', '')
-        if plugin_paths:
-            wa_config.set('plugin_paths', plugin_paths.split(os.pathsep))
-        ext_paths = environ.pop('WA_EXTENSION_PATHS', '')
-        if ext_paths:
-            wa_config.set('plugin_paths', ext_paths.split(os.pathsep))
-
-
 # Command line options are parsed in the "run" command. This is used to send
 # certain arguments to the correct configuration points and keep a record of
 # how WA was invoked
 class CommandLineArgsParser(object):
+
     def __init__(self, cmd_args, wa_config, jobs_config):
         wa_config.set("verbosity", cmd_args.verbosity)
         # TODO: Is this correct? Does there need to be a third output dir param

wlauto/core/entry_point.py

@@ -24,6 +24,7 @@ import warnings
 from wlauto.core.configuration import settings
 from wlauto.core import pluginloader
 from wlauto.core.command import init_argument_parser
+from wlauto.core.host import init_user_directory
 from wlauto.exceptions import WAError, ConfigError
 from wlauto.utils.misc import get_traceback
 from wlauto.utils.log import init_logging
@@ -45,7 +46,11 @@ def load_commands(subparsers):
 
 
 def main():
+    if not os.path.exists(settings.user_directory):
+        init_user_directory()
+
     try:
+
         description = ("Execute automated workloads on a remote device and process "
                        "the resulting output.\n\nUse \"wa <subcommand> -h\" to see "
                        "help for individual subcommands.")
@@ -57,10 +62,7 @@ def main():
         commands = load_commands(parser.add_subparsers(dest='command'))  # each command will add its own subparser
         args = parser.parse_args()
 
-        #TODO: Set this stuff properly, i.e dont use settings (if possible)
+        settings.set("verbosity", args.verbose)
-        #settings.set("verbosity", args.verbose)
-        #settings.load_user_config()
-        #settings.debug = args.debug
 
         for config in args.config:
             if not os.path.exists(config):

wlauto/core/host.py

@@ -0,0 +1,33 @@
+import os
+
+from wlauto.core.configuration import settings
+
+def init_user_directory(overwrite_existing=False):  # pylint: disable=R0914
+    """
+    Initialise a fresh user directory. 
+    """
+    if os.path.exists(settings.user_directory):
+        if not overwrite_existing:
+            raise RuntimeError('Environment {} already exists.'.format(settings.user_directory))
+        shutil.rmtree(settings.user_directory)
+
+    os.makedirs(settings.user_directory)
+    os.makedirs(settings.dependencies_directory)
+    os.makedirs(settings.plugins_directory)
+
+    # TODO: generate default config.yaml here
+
+    if os.getenv('USER') == 'root':
+        # If running with sudo on POSIX, change the ownership to the real user.
+        real_user = os.getenv('SUDO_USER')
+        if real_user:
+            import pwd  # done here as module won't import on win32
+            user_entry = pwd.getpwnam(real_user)
+            uid, gid = user_entry.pw_uid, user_entry.pw_gid
+            os.chown(settings.user_directory, uid, gid)
+            # why, oh why isn't there a recusive=True option for os.chown?
+            for root, dirs, files in os.walk(settings.user_directory):
+                for d in dirs:
+                    os.chown(os.path.join(root, d), uid, gid)
+                for f in files: 
+                    os.chown(os.path.join(root, f), uid, gid)

wlauto/core/plugin.py

@@ -25,13 +25,14 @@ from collections import OrderedDict, defaultdict
 from itertools import chain
 from copy import copy
 
-from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError
+from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError, HostError
 from wlauto.utils.misc import (ensure_directory_exists as _d,
                                walk_modules, load_class, merge_dicts_simple, get_article)
 from wlauto.core.configuration import settings
 from wlauto.utils.types import identifier, boolean
 from wlauto.core.configuration.configuration import ConfigurationPoint as Parameter
 
+
 MODNAME_TRANS = string.maketrans(':/\\.', '____')
 
 
@@ -697,10 +698,9 @@ class PluginLoader(object):
             for package in packages:
                 for module in walk_modules(package):
                     self._discover_in_module(module)
-        except ImportError as e:
+        except HostError as e:
-            source = getattr(e, 'path', package)
             message = 'Problem loading plugins from {}: {}'
-            raise LoaderError(message.format(source, e.message))
+            raise LoaderError(message.format(e.module, str(e.orig_exc)))
 
     def _discover_from_paths(self, paths, ignore_paths):
         paths = paths or []

wlauto/core/pluginloader.py

@@ -38,8 +38,7 @@ class __LoaderWrapper(object):
         from wlauto.core.plugin import PluginLoader
         from wlauto.core.configuration import settings
         self._loader = PluginLoader(settings.plugin_packages,
-                                    settings.plugin_paths,
+                                    [settings.plugins_directory], [])
-                                    settings.plugin_ignore_paths)
 
     def update(self, packages=None, paths=None, ignore_paths=None):
         if not self._loader:

wlauto/exceptions.py

@@ -14,7 +14,9 @@
 #
 
 
-from wlauto.utils.misc import get_traceback, TimeoutError  # NOQA pylint: disable=W0611
+from wlauto.utils.misc import get_traceback
+
+from devlib.exception import DevlibError, HostError, TargetError, TimeoutError
 
 
 class WAError(Exception):

wlauto/tests/test_parsers.py

@@ -8,8 +8,8 @@ from mock.mock import Mock, MagicMock, call
 from wlauto.exceptions import ConfigError
 from wlauto.core.configuration.parsers import *  # pylint: disable=wildcard-import
 from wlauto.core.configuration.parsers import _load_file, _collect_valid_id, _resolve_params_alias
-from wlauto.core.configuration import (WAConfiguration, RunConfiguration, JobGenerator,
+from wlauto.core.configuration import RunConfiguration, JobGenerator, PluginCache, ConfigurationPoint
-                                       PluginCache, ConfigurationPoint)
+from wlauto.core.configuration.configuration import WAConfiguration
 from wlauto.utils.types import toggle_set, reset_counter
 
 
@@ -125,9 +125,6 @@ class TestFunctions(TestCase):
         with self.assertRaises(ConfigError):
             _resolve_params_alias(test, "new_name")
 
-    def test_construct_valid_entry(self):
-        raise Exception()
-
 
 class TestConfigParser(TestCase):
 
@@ -362,44 +359,6 @@ class TestAgendaParser(TestCase):
         assert_equal(workload['workload_name'], "test")
 
 
-class TestEnvironmentVarsParser(TestCase):
-
-    def test_environmentvarsparser(self):
-        wa_config = Mock(spec=WAConfiguration)
-        calls = [call('user_directory', '/testdir'),
-                 call('plugin_paths', ['/test', '/some/other/path', '/testy/mc/test/face'])]
-
-        # Valid env vars
-        valid_environ = {"WA_USER_DIRECTORY": "/testdir",
-                         "WA_PLUGIN_PATHS": "/test:/some/other/path:/testy/mc/test/face"}
-        EnvironmentVarsParser(wa_config, valid_environ)
-        wa_config.set.assert_has_calls(calls)
-
-        # Alternative env var name
-        wa_config.reset_mock()
-        alt_valid_environ = {"WA_USER_DIRECTORY": "/testdir",
-                             "WA_EXTENSION_PATHS": "/test:/some/other/path:/testy/mc/test/face"}
-        EnvironmentVarsParser(wa_config, alt_valid_environ)
-        wa_config.set.assert_has_calls(calls)
-
-        # Test that WA_EXTENSION_PATHS gets merged with WA_PLUGIN_PATHS.
-        # Also checks that other enviroment variables don't cause errors
-        wa_config.reset_mock()
-        calls = [call('user_directory', '/testdir'),
-                 call('plugin_paths', ['/test', '/some/other/path']),
-                 call('plugin_paths', ['/testy/mc/test/face'])]
-        ext_and_plgin = {"WA_USER_DIRECTORY": "/testdir",
-                         "WA_PLUGIN_PATHS": "/test:/some/other/path",
-                         "WA_EXTENSION_PATHS": "/testy/mc/test/face",
-                         "RANDOM_VAR": "random_value"}
-        EnvironmentVarsParser(wa_config, ext_and_plgin)
-        # If any_order=True then the calls can be in any order, but they must all appear
-        wa_config.set.assert_has_calls(calls, any_order=True)
-
-        # No WA enviroment variables present
-        wa_config.reset_mock()
-        EnvironmentVarsParser(wa_config, {"RANDOM_VAR": "random_value"})
-        wa_config.set.assert_not_called()
 
 
 class TestCommandLineArgsParser(TestCase):

wlauto/utils/misc.py

@@ -492,7 +492,7 @@ def merge_config_values(base, other):
            are treated as atomic, and not mergeable.
         s: A sequence. Anything iterable that is not a dict or
            a string (strings are considered scalars).
-        m: A key-value mapping. ``dict`` and it's derivatives.
+        m: A key-value mapping. ``dict`` and its derivatives.
         n: ``None``.
         o: A mergeable object; this is an object that implements both
           ``merge_with`` and ``merge_into`` methods.

wlauto/utils/serializer.py

@@ -51,7 +51,7 @@ import yaml as _yaml
 import dateutil.parser
 
 from wlauto.exceptions import SerializerSyntaxError
-from wlauto.utils.types import regex_type
+from wlauto.utils.types import regex_type, none_type
 from wlauto.utils.misc import isiterable
 
 
@@ -70,12 +70,14 @@ POD_TYPES = [
     tuple,
     dict,
     set,
-    basestring,
+    str,
+    unicode,
     int,
     float,
     bool,
     datetime,
-    regex_type
+    regex_type,
+    none_type,
 ]
 
 class WAJSONEncoder(_json.JSONEncoder):
@@ -257,3 +259,4 @@ def _read_pod(fh, fmt=None):
 
 def is_pod(obj):
     return type(obj) in POD_TYPES
+

wlauto/utils/types.py

@@ -169,6 +169,7 @@ list_or_bool = list_or(boolean)
 
 
 regex_type = type(re.compile(''))
+none_type = type(None)
 
 
 def regex(value):