sascha/workload-automation
1
0
Fork 0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-01-18 20:11:20 +00:00
Code Issues Releases Wiki Activity

Fixing things up to a point where "list" and "show" commands work.

Browse Source
This commit is contained in:
Sergei Trofimov 2017-02-09 09:09:00 +00:00
parent 92aeca8125
commit dc6d9676f2
15 changed files with 115 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -78,7 +78,7 @@ params = dict(
'pyYAML', # YAML-formatted agenda parsing
'requests', # Fetch assets over HTTP
'devlib', # Interacting with devices
'louie' # Handles signal callbacks
'louie' # callbacks dispatch
],
extras_require={
'other': ['jinja2', 'pandas>=0.13.1'],

6
wlauto/commands/run.py
View File

@ -22,9 +22,9 @@ import wlauto
from wlauto import Command, settings
from wlauto.core.execution import Executor
from wlauto.utils.log import add_log_file
from wlauto.core.configuration import RunConfiguration, WAConfiguration
from wlauto.core.configuration import RunConfiguration
from wlauto.core import pluginloader
from wlauto.core.configuration_parsers import Agenda, ConfigFile, EnvrironmentVars, CommandLineArgs
from wlauto.core.configuration.parsers import AgendaParser, ConfigParser, CommandLineArgsParser
class RunCommand(Command):
@ -74,7 +74,7 @@ class RunCommand(Command):
# STAGE 1: Gather configuratation
env = EnvrironmentVars()
env = EnvironmentVars()
args = CommandLineArgs(args)
# STAGE 2.1a: Early WAConfiguration, required to find config files

2
wlauto/core/command.py
View File

@ -21,7 +21,7 @@ from wlauto.core.version import get_wa_version
def init_argument_parser(parser):
parser.add_argument('-c', '--config', help='specify an additional config.py', action='append')
parser.add_argument('-c', '--config', help='specify an additional config.py', action='append', default=[])
parser.add_argument('-v', '--verbose', action='count',
help='The scripts will produce verbose output.')
parser.add_argument('--version', action='version', version='%(prog)s {}'.format(get_wa_version()))

1
wlauto/core/configuration/__init__.py
View File

@ -13,7 +13,6 @@
# limitations under the License.
#
from wlauto.core.configuration.configuration import (settings,
WAConfiguration,
RunConfiguration,
JobGenerator,
ConfigurationPoint)

124
wlauto/core/configuration/configuration.py
View File

@ -492,36 +492,11 @@ class CpuFreqParameters(object):
class Configuration(object):
config_points = []
name = ""
name = ''
# The below line must be added to all subclasses
configuration = {cp.name: cp for cp in config_points}
def __init__(self):
# Load default values for configuration points
for confpoint in self.configuration.itervalues():
confpoint.set_value(self, check_mandatory=False)
def set(self, name, value, check_mandatory=True):
if name not in self.configuration:
raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
def update_config(self, values, check_mandatory=True):
for k, v in values.iteritems():
self.set(k, v, check_mandatory=check_mandatory)
def validate(self):
for cfg_point in self.configuration.itervalues():
cfg_point.validate(self)
def to_pod(self):
pod = {}
for cfg_point_name in self.configuration.iterkeys():
value = getattr(self, cfg_point_name, None)
if value is not None:
pod[cfg_point_name] = value
return pod
@classmethod
# pylint: disable=unused-argument
def from_pod(cls, pod, plugin_cache):
@ -535,11 +510,46 @@ class Configuration(object):
instance.validate()
return instance
def __init__(self):
for confpoint in self.config_points:
confpoint.set_value(self, check_mandatory=False)
def set(self, name, value, check_mandatory=True):
if name not in self.configuration:
raise ConfigError('Unknown {} configuration "{}"'.format(self.name, name))
self.configuration[name].set_value(self, value, check_mandatory=check_mandatory)
def update_config(self, values, check_mandatory=True):
for k, v in values.iteritems():
self.set(k, v, check_mandatory=check_mandatory)
def validate(self):
for cfg_point in self.config_points:
cfg_point.validate(self)
def to_pod(self):
pod = {}
for cfg_point_name in self.configuration.iterkeys():
value = getattr(self, cfg_point_name, None)
if value is not None:
pod[cfg_point_name] = value
return pod
# This configuration for the core WA framework
class WAConfiguration(Configuration):
name = "WA Configuration"
plugin_packages = [
'wlauto.commands',
'wlauto.workloads',
'wlauto.instrumentation',
'wlauto.result_processors',
'wlauto.managers',
'wlauto.resource_getters',
]
config_points = [
ConfigurationPoint(
'user_directory',
@ -550,48 +560,6 @@ class WAConfiguration(Configuration):
kind=str,
default=os.path.join(os.path.expanduser('~'), '.workload_automation'),
),
ConfigurationPoint(
'plugin_packages',
kind=list_of_strings,
default=[
'wlauto.commands',
'wlauto.workloads',
'wlauto.instrumentation',
'wlauto.result_processors',
'wlauto.managers',
'wlauto.resource_getters',
],
description="""
List of packages that will be scanned for WA plugins.
""",
),
ConfigurationPoint(
'plugin_paths',
kind=list_of_strings,
default=[
'workloads',
'instruments',
'targets',
'processors',
# Legacy
'managers',
'result_processors',
],
description="""
List of paths that will be scanned for WA plugins.
""",
merge=True
),
ConfigurationPoint(
'plugin_ignore_paths',
kind=list_of_strings,
default=[],
description="""
List of (sub)paths that will be ignored when scanning
``plugin_paths`` for WA plugins.
""",
),
ConfigurationPoint(
'assets_repository',
description="""
@ -623,7 +591,7 @@ class WAConfiguration(Configuration):
Verbosity of console output.
""",
),
ConfigurationPoint( # TODO: Needs some format for dates ect/ comes from cfg
ConfigurationPoint( # TODO: Needs some format for dates etc/ comes from cfg
'default_output_directory',
default="wa_output",
description="""
@ -636,7 +604,19 @@ class WAConfiguration(Configuration):
@property
def dependencies_directory(self):
return "{}/dependencies/".format(self.user_directory)
return os.path.join(self.user_directory, 'dependencies')
@property
def plugins_directory(self):
return os.path.join(self.user_directory, 'plugins')
def __init__(self, environ):
super(WAConfiguration, self).__init__()
user_directory = environ.pop('WA_USER_DIRECTORY', '')
if user_directory:
self.set('user_directory', user_directory)
# This is generic top-level configuration for WA runs.
@ -1029,4 +1009,4 @@ class JobGenerator(object):
yield job_spec
settings = WAConfiguration()
settings = WAConfiguration(os.environ)

14
wlauto/core/configuration/parsers.py
View File

@ -283,23 +283,11 @@ class AgendaParser(object):
raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
class EnvironmentVarsParser(object):
def __init__(self, wa_config, environ):
user_directory = environ.pop('WA_USER_DIRECTORY', '')
if user_directory:
wa_config.set('user_directory', user_directory)
plugin_paths = environ.pop('WA_PLUGIN_PATHS', '')
if plugin_paths:
wa_config.set('plugin_paths', plugin_paths.split(os.pathsep))
ext_paths = environ.pop('WA_EXTENSION_PATHS', '')
if ext_paths:
wa_config.set('plugin_paths', ext_paths.split(os.pathsep))
# Command line options are parsed in the "run" command. This is used to send
# certain arguments to the correct configuration points and keep a record of
# how WA was invoked
class CommandLineArgsParser(object):
def __init__(self, cmd_args, wa_config, jobs_config):
wa_config.set("verbosity", cmd_args.verbosity)
# TODO: Is this correct? Does there need to be a third output dir param

10
wlauto/core/entry_point.py
View File

@ -24,6 +24,7 @@ import warnings
from wlauto.core.configuration import settings
from wlauto.core import pluginloader
from wlauto.core.command import init_argument_parser
from wlauto.core.host import init_user_directory
from wlauto.exceptions import WAError, ConfigError
from wlauto.utils.misc import get_traceback
from wlauto.utils.log import init_logging
@ -45,7 +46,11 @@ def load_commands(subparsers):
def main():
if not os.path.exists(settings.user_directory):
init_user_directory()
try:
description = ("Execute automated workloads on a remote device and process "
"the resulting output.\n\nUse \"wa <subcommand> -h\" to see "
"help for individual subcommands.")
@ -57,10 +62,7 @@ def main():
commands = load_commands(parser.add_subparsers(dest='command')) # each command will add its own subparser
args = parser.parse_args()
#TODO: Set this stuff properly, i.e dont use settings (if possible)
#settings.set("verbosity", args.verbose)
#settings.load_user_config()
#settings.debug = args.debug
settings.set("verbosity", args.verbose)
for config in args.config:
if not os.path.exists(config):

33
wlauto/core/host.py Normal file
View File

@ -0,0 +1,33 @@
import os
from wlauto.core.configuration import settings
def init_user_directory(overwrite_existing=False): # pylint: disable=R0914
"""
Initialise a fresh user directory.
"""
if os.path.exists(settings.user_directory):
if not overwrite_existing:
raise RuntimeError('Environment {} already exists.'.format(settings.user_directory))
shutil.rmtree(settings.user_directory)
os.makedirs(settings.user_directory)
os.makedirs(settings.dependencies_directory)
os.makedirs(settings.plugins_directory)
# TODO: generate default config.yaml here
if os.getenv('USER') == 'root':
# If running with sudo on POSIX, change the ownership to the real user.
real_user = os.getenv('SUDO_USER')
if real_user:
import pwd # done here as module won't import on win32
user_entry = pwd.getpwnam(real_user)
uid, gid = user_entry.pw_uid, user_entry.pw_gid
os.chown(settings.user_directory, uid, gid)
# why, oh why isn't there a recusive=True option for os.chown?
for root, dirs, files in os.walk(settings.user_directory):
for d in dirs:
os.chown(os.path.join(root, d), uid, gid)
for f in files:
os.chown(os.path.join(root, f), uid, gid)

8
wlauto/core/plugin.py
View File

@ -25,13 +25,14 @@ from collections import OrderedDict, defaultdict
from itertools import chain
from copy import copy
from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError
from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError, HostError
from wlauto.utils.misc import (ensure_directory_exists as _d,
walk_modules, load_class, merge_dicts_simple, get_article)
from wlauto.core.configuration import settings
from wlauto.utils.types import identifier, boolean
from wlauto.core.configuration.configuration import ConfigurationPoint as Parameter
MODNAME_TRANS = string.maketrans(':/\\.', '____')
@ -697,10 +698,9 @@ class PluginLoader(object):
for package in packages:
for module in walk_modules(package):
self._discover_in_module(module)
except ImportError as e:
source = getattr(e, 'path', package)
except HostError as e:
message = 'Problem loading plugins from {}: {}'
raise LoaderError(message.format(source, e.message))
raise LoaderError(message.format(e.module, str(e.orig_exc)))
def _discover_from_paths(self, paths, ignore_paths):
paths = paths or []

3
wlauto/core/pluginloader.py
View File

@ -38,8 +38,7 @@ class __LoaderWrapper(object):
from wlauto.core.plugin import PluginLoader
from wlauto.core.configuration import settings
self._loader = PluginLoader(settings.plugin_packages,
settings.plugin_paths,
settings.plugin_ignore_paths)
[settings.plugins_directory], [])
def update(self, packages=None, paths=None, ignore_paths=None):
if not self._loader:

4
wlauto/exceptions.py
View File

@ -14,7 +14,9 @@
#
from wlauto.utils.misc import get_traceback, TimeoutError # NOQA pylint: disable=W0611
from wlauto.utils.misc import get_traceback
from devlib.exception import DevlibError, HostError, TargetError, TimeoutError
class WAError(Exception):

45
wlauto/tests/test_parsers.py
View File

@ -8,8 +8,8 @@ from mock.mock import Mock, MagicMock, call
from wlauto.exceptions import ConfigError
from wlauto.core.configuration.parsers import * # pylint: disable=wildcard-import
from wlauto.core.configuration.parsers import _load_file, _collect_valid_id, _resolve_params_alias
from wlauto.core.configuration import (WAConfiguration, RunConfiguration, JobGenerator,
PluginCache, ConfigurationPoint)
from wlauto.core.configuration import RunConfiguration, JobGenerator, PluginCache, ConfigurationPoint
from wlauto.core.configuration.configuration import WAConfiguration
from wlauto.utils.types import toggle_set, reset_counter
@ -125,9 +125,6 @@ class TestFunctions(TestCase):
with self.assertRaises(ConfigError):
_resolve_params_alias(test, "new_name")
def test_construct_valid_entry(self):
raise Exception()
class TestConfigParser(TestCase):
@ -362,44 +359,6 @@ class TestAgendaParser(TestCase):
assert_equal(workload['workload_name'], "test")
class TestEnvironmentVarsParser(TestCase):
def test_environmentvarsparser(self):
wa_config = Mock(spec=WAConfiguration)
calls = [call('user_directory', '/testdir'),
call('plugin_paths', ['/test', '/some/other/path', '/testy/mc/test/face'])]
# Valid env vars
valid_environ = {"WA_USER_DIRECTORY": "/testdir",
"WA_PLUGIN_PATHS": "/test:/some/other/path:/testy/mc/test/face"}
EnvironmentVarsParser(wa_config, valid_environ)
wa_config.set.assert_has_calls(calls)
# Alternative env var name
wa_config.reset_mock()
alt_valid_environ = {"WA_USER_DIRECTORY": "/testdir",
"WA_EXTENSION_PATHS": "/test:/some/other/path:/testy/mc/test/face"}
EnvironmentVarsParser(wa_config, alt_valid_environ)
wa_config.set.assert_has_calls(calls)
# Test that WA_EXTENSION_PATHS gets merged with WA_PLUGIN_PATHS.
# Also checks that other enviroment variables don't cause errors
wa_config.reset_mock()
calls = [call('user_directory', '/testdir'),
call('plugin_paths', ['/test', '/some/other/path']),
call('plugin_paths', ['/testy/mc/test/face'])]
ext_and_plgin = {"WA_USER_DIRECTORY": "/testdir",
"WA_PLUGIN_PATHS": "/test:/some/other/path",
"WA_EXTENSION_PATHS": "/testy/mc/test/face",
"RANDOM_VAR": "random_value"}
EnvironmentVarsParser(wa_config, ext_and_plgin)
# If any_order=True then the calls can be in any order, but they must all appear
wa_config.set.assert_has_calls(calls, any_order=True)
# No WA enviroment variables present
wa_config.reset_mock()
EnvironmentVarsParser(wa_config, {"RANDOM_VAR": "random_value"})
wa_config.set.assert_not_called()
class TestCommandLineArgsParser(TestCase):

2
wlauto/utils/misc.py
View File

@ -492,7 +492,7 @@ def merge_config_values(base, other):
are treated as atomic, and not mergeable.
s: A sequence. Anything iterable that is not a dict or
a string (strings are considered scalars).
m: A key-value mapping. ``dict`` and it's derivatives.
m: A key-value mapping. ``dict`` and its derivatives.
n: ``None``.
o: A mergeable object; this is an object that implements both
``merge_with`` and ``merge_into`` methods.

9
wlauto/utils/serializer.py
View File

@ -51,7 +51,7 @@ import yaml as _yaml
import dateutil.parser
from wlauto.exceptions import SerializerSyntaxError
from wlauto.utils.types import regex_type
from wlauto.utils.types import regex_type, none_type
from wlauto.utils.misc import isiterable
@ -70,12 +70,14 @@ POD_TYPES = [
tuple,
dict,
set,
basestring,
str,
unicode,
int,
float,
bool,
datetime,
regex_type
regex_type,
none_type,
]
class WAJSONEncoder(_json.JSONEncoder):
@ -257,3 +259,4 @@ def _read_pod(fh, fmt=None):
def is_pod(obj):
return type(obj) in POD_TYPES

1
wlauto/utils/types.py
View File

@ -169,6 +169,7 @@ list_or_bool = list_or(boolean)
regex_type = type(re.compile(''))
none_type = type(None)
def regex(value):