Fixing things up to a point where "list" and "show" commands work.

wlauto/core/command.py

@@ -21,7 +21,7 @@ from wlauto.core.version import get_wa_version
 
 
 def init_argument_parser(parser):
-    parser.add_argument('-c', '--config', help='specify an additional config.py', action='append')
+    parser.add_argument('-c', '--config', help='specify an additional config.py', action='append', default=[])
     parser.add_argument('-v', '--verbose', action='count',
                         help='The scripts will produce verbose output.')
     parser.add_argument('--version', action='version', version='%(prog)s {}'.format(get_wa_version()))

wlauto/core/configuration/__init__.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
 from wlauto.core.configuration.configuration import (settings,
-                                                     WAConfiguration,
                                                      RunConfiguration,
                                                      JobGenerator,
                                                      ConfigurationPoint)

wlauto/core/configuration/configuration.py

@@ -492,36 +492,11 @@ class CpuFreqParameters(object):
 class Configuration(object):
 
     config_points = []
-    name = ""
+    name = ''
+
     # The below line must be added to all subclasses
     configuration = {cp.name: cp for cp in config_points}
 
-    def __init__(self):
-        # Load default values for configuration points
-        for confpoint in self.configuration.itervalues():
-            confpoint.set_value(self, check_mandatory=False)
-
-    def set(self, name, value, check_mandatory=True):
-        if name not in self.configuration:
-            raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
-        self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
-
-    def update_config(self, values, check_mandatory=True):
-        for k, v in values.iteritems():
-            self.set(k, v, check_mandatory=check_mandatory)
-
-    def validate(self):
-        for cfg_point in self.configuration.itervalues():
-            cfg_point.validate(self)
-
-    def to_pod(self):
-        pod = {}
-        for cfg_point_name in self.configuration.iterkeys():
-            value = getattr(self, cfg_point_name, None)
-            if value is not None:
-                pod[cfg_point_name] = value
-        return pod
-
     @classmethod
     # pylint: disable=unused-argument
     def from_pod(cls, pod, plugin_cache):
@@ -535,11 +510,46 @@ class Configuration(object):
         instance.validate()
         return instance
 
+    def __init__(self):
+        for confpoint in self.config_points:
+            confpoint.set_value(self, check_mandatory=False)
+
+    def set(self, name, value, check_mandatory=True):
+        if name not in self.configuration:
+            raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
+        self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
+
+    def update_config(self, values, check_mandatory=True):
+        for k, v in values.iteritems():
+            self.set(k, v, check_mandatory=check_mandatory)
+
+    def validate(self):
+        for cfg_point in self.config_points:
+            cfg_point.validate(self)
+
+    def to_pod(self):
+        pod = {}
+        for cfg_point_name in self.configuration.iterkeys():
+            value = getattr(self, cfg_point_name, None)
+            if value is not None:
+                pod[cfg_point_name] = value
+        return pod
+
 
 # This configuration for the core WA framework
 class WAConfiguration(Configuration):
 
     name = "WA Configuration"
+
+    plugin_packages = [
+        'wlauto.commands',
+        'wlauto.workloads',
+        'wlauto.instrumentation',
+        'wlauto.result_processors',
+        'wlauto.managers',
+        'wlauto.resource_getters',
+    ]
+
     config_points = [
         ConfigurationPoint(
             'user_directory',
@@ -550,48 +560,6 @@ class WAConfiguration(Configuration):
             kind=str,
             default=os.path.join(os.path.expanduser('~'), '.workload_automation'),
         ),
-        ConfigurationPoint(
-            'plugin_packages',
-            kind=list_of_strings,
-            default=[
-                'wlauto.commands',
-                'wlauto.workloads',
-                'wlauto.instrumentation',
-                'wlauto.result_processors',
-                'wlauto.managers',
-                'wlauto.resource_getters',
-            ],
-            description="""
-            List of packages that will be scanned for WA plugins.
-            """,
-        ),
-        ConfigurationPoint(
-            'plugin_paths',
-            kind=list_of_strings,
-            default=[
-                'workloads',
-                'instruments',
-                'targets',
-                'processors',
-
-                # Legacy
-                'managers',
-                'result_processors',
-            ],
-            description="""
-            List of paths that will be scanned for WA plugins.
-            """,
-            merge=True
-        ),
-        ConfigurationPoint(
-            'plugin_ignore_paths',
-            kind=list_of_strings,
-            default=[],
-            description="""
-            List of (sub)paths that will be ignored when scanning
-            ``plugin_paths`` for WA plugins.
-            """,
-        ),
         ConfigurationPoint(
             'assets_repository',
             description="""
@@ -623,7 +591,7 @@ class WAConfiguration(Configuration):
             Verbosity of console output.
             """,
         ),
-        ConfigurationPoint(  # TODO: Needs some format for dates ect/ comes from cfg
+        ConfigurationPoint(  # TODO: Needs some format for dates etc/ comes from cfg
             'default_output_directory',
             default="wa_output",
             description="""
@@ -636,7 +604,19 @@ class WAConfiguration(Configuration):
 
     @property
     def dependencies_directory(self):
-        return "{}/dependencies/".format(self.user_directory)
+        return os.path.join(self.user_directory, 'dependencies')
+
+    @property
+    def plugins_directory(self):
+        return os.path.join(self.user_directory, 'plugins')
+
+
+    def __init__(self, environ):
+        super(WAConfiguration, self).__init__()
+        user_directory = environ.pop('WA_USER_DIRECTORY', '')
+        if user_directory:
+            self.set('user_directory', user_directory)
+
 
 
 # This is generic top-level configuration for WA runs.
@@ -1029,4 +1009,4 @@ class JobGenerator(object):
 
                 yield job_spec
 
-settings = WAConfiguration()
+settings = WAConfiguration(os.environ)

wlauto/core/configuration/parsers.py

@@ -283,23 +283,11 @@ class AgendaParser(object):
             raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
 
 
-class EnvironmentVarsParser(object):
-    def __init__(self, wa_config, environ):
-        user_directory = environ.pop('WA_USER_DIRECTORY', '')
-        if user_directory:
-            wa_config.set('user_directory', user_directory)
-        plugin_paths = environ.pop('WA_PLUGIN_PATHS', '')
-        if plugin_paths:
-            wa_config.set('plugin_paths', plugin_paths.split(os.pathsep))
-        ext_paths = environ.pop('WA_EXTENSION_PATHS', '')
-        if ext_paths:
-            wa_config.set('plugin_paths', ext_paths.split(os.pathsep))
-
-
 # Command line options are parsed in the "run" command. This is used to send
 # certain arguments to the correct configuration points and keep a record of
 # how WA was invoked
 class CommandLineArgsParser(object):
+
     def __init__(self, cmd_args, wa_config, jobs_config):
         wa_config.set("verbosity", cmd_args.verbosity)
         # TODO: Is this correct? Does there need to be a third output dir param

wlauto/core/entry_point.py

@@ -24,6 +24,7 @@ import warnings
 from wlauto.core.configuration import settings
 from wlauto.core import pluginloader
 from wlauto.core.command import init_argument_parser
+from wlauto.core.host import init_user_directory
 from wlauto.exceptions import WAError, ConfigError
 from wlauto.utils.misc import get_traceback
 from wlauto.utils.log import init_logging
@@ -45,7 +46,11 @@ def load_commands(subparsers):
 
 
 def main():
+    if not os.path.exists(settings.user_directory):
+        init_user_directory()
+
     try:
+
         description = ("Execute automated workloads on a remote device and process "
                        "the resulting output.\n\nUse \"wa <subcommand> -h\" to see "
                        "help for individual subcommands.")
@@ -57,10 +62,7 @@ def main():
         commands = load_commands(parser.add_subparsers(dest='command'))  # each command will add its own subparser
         args = parser.parse_args()
 
-        #TODO: Set this stuff properly, i.e dont use settings (if possible)
-        #settings.set("verbosity", args.verbose)
-        #settings.load_user_config()
-        #settings.debug = args.debug
+        settings.set("verbosity", args.verbose)
 
         for config in args.config:
             if not os.path.exists(config):

wlauto/core/host.py

@@ -0,0 +1,33 @@
+import os
+
+from wlauto.core.configuration import settings
+
+def init_user_directory(overwrite_existing=False):  # pylint: disable=R0914
+    """
+    Initialise a fresh user directory. 
+    """
+    if os.path.exists(settings.user_directory):
+        if not overwrite_existing:
+            raise RuntimeError('Environment {} already exists.'.format(settings.user_directory))
+        shutil.rmtree(settings.user_directory)
+
+    os.makedirs(settings.user_directory)
+    os.makedirs(settings.dependencies_directory)
+    os.makedirs(settings.plugins_directory)
+
+    # TODO: generate default config.yaml here
+
+    if os.getenv('USER') == 'root':
+        # If running with sudo on POSIX, change the ownership to the real user.
+        real_user = os.getenv('SUDO_USER')
+        if real_user:
+            import pwd  # done here as module won't import on win32
+            user_entry = pwd.getpwnam(real_user)
+            uid, gid = user_entry.pw_uid, user_entry.pw_gid
+            os.chown(settings.user_directory, uid, gid)
+            # why, oh why isn't there a recusive=True option for os.chown?
+            for root, dirs, files in os.walk(settings.user_directory):
+                for d in dirs:
+                    os.chown(os.path.join(root, d), uid, gid)
+                for f in files: 
+                    os.chown(os.path.join(root, f), uid, gid)

wlauto/core/plugin.py

@@ -25,13 +25,14 @@ from collections import OrderedDict, defaultdict
 from itertools import chain
 from copy import copy
 
-from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError
+from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError, HostError
 from wlauto.utils.misc import (ensure_directory_exists as _d,
                                walk_modules, load_class, merge_dicts_simple, get_article)
 from wlauto.core.configuration import settings
 from wlauto.utils.types import identifier, boolean
 from wlauto.core.configuration.configuration import ConfigurationPoint as Parameter
 
+
 MODNAME_TRANS = string.maketrans(':/\\.', '____')
 
 
@@ -697,10 +698,9 @@ class PluginLoader(object):
             for package in packages:
                 for module in walk_modules(package):
                     self._discover_in_module(module)
-        except ImportError as e:
-            source = getattr(e, 'path', package)
+        except HostError as e:
             message = 'Problem loading plugins from {}: {}'
-            raise LoaderError(message.format(source, e.message))
+            raise LoaderError(message.format(e.module, str(e.orig_exc)))
 
     def _discover_from_paths(self, paths, ignore_paths):
         paths = paths or []

wlauto/core/pluginloader.py

@@ -38,8 +38,7 @@ class __LoaderWrapper(object):
         from wlauto.core.plugin import PluginLoader
         from wlauto.core.configuration import settings
         self._loader = PluginLoader(settings.plugin_packages,
-                                    settings.plugin_paths,
-                                    settings.plugin_ignore_paths)
+                                    [settings.plugins_directory], [])
 
     def update(self, packages=None, paths=None, ignore_paths=None):
         if not self._loader: