Add WA Create Database Command

Add a command to create a PostgreSQL database with supplied parameters
which draws its structure from the supplied schema (Version 1.1). This
database is of a format intended to be used with the forthcoming WA
Postgres output processor.

setup.py

@@ -31,7 +31,7 @@ wa_dir = os.path.join(os.path.dirname(__file__), 'wa')
 sys.path.insert(0, os.path.join(wa_dir, 'framework'))
 from version import get_wa_version, get_wa_version_with_commit
 
-# happends if falling back to distutils
+# happens if falling back to distutils
 warnings.filterwarnings('ignore', "Unknown distribution option: 'install_requires'")
 warnings.filterwarnings('ignore', "Unknown distribution option: 'extras_require'")
 
@@ -41,7 +41,7 @@ except OSError:
     pass
 
 packages = []
-data_files = {}
+data_files = {'': [os.path.join(wa_dir, 'commands', 'postgres_schema.sql')]}
 source_dir = os.path.dirname(__file__)
 for root, dirs, files in os.walk(wa_dir):
     rel_dir = os.path.relpath(root, source_dir)
@@ -67,6 +67,7 @@ params = dict(
     version=get_wa_version_with_commit(),
     packages=packages,
     package_data=data_files,
+    include_package_data=True,
     scripts=scripts,
     url='https://github.com/ARM-software/workload-automation',
     license='Apache v2',

wa/commands/create.py

@@ -13,16 +13,26 @@
 # limitations under the License.
 #
 
+
 import os
 import sys
 import stat
 import shutil
 import string
+import re
+import uuid
 import getpass
 from collections import OrderedDict
 from distutils.dir_util import copy_tree  # pylint: disable=no-name-in-module, import-error
 
 from devlib.utils.types import identifier
+try:
+    import psycopg2
+    from psycopg2 import connect, OperationalError, extras
+    from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+except ImportError as e:
+    psycopg2 = None
+    import_error_msg = e.args[0] if e.args else str(e)
 
 from wa import ComplexCommand, SubCommand, pluginloader, settings
 from wa.framework.target.descriptor import list_target_descriptions
@@ -36,6 +46,142 @@ from wa.utils.serializer import yaml
 TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates')
 
 
+class CreateDatabaseSubcommand(SubCommand):
+
+    name = 'database'
+    description = """
+    Create a Postgresql database which is compatible with the WA Postgres
+    output processor.
+    """
+
+    schemafilepath = 'postgres_schema.sql'
+
+    def __init__(self, *args, **kwargs):
+        super(CreateDatabaseSubcommand, self).__init__(*args, **kwargs)
+        self.sql_commands = None
+        self.schemaversion = None
+        self.schema_major = None
+        self.schema_minor = None
+
+    def initialize(self, context):
+        self.parser.add_argument(
+            '-a', '--postgres-host', default='localhost',
+            help='The host on which to create the database.')
+        self.parser.add_argument(
+            '-k', '--postgres-port', default='5432',
+            help='The port on which the PostgreSQL server is running.')
+        self.parser.add_argument(
+            '-u', '--username', default='postgres',
+            help='The username with which to connect to the server.')
+        self.parser.add_argument(
+            '-p', '--password',
+            help='The password for the user account.')
+        self.parser.add_argument(
+            '-d', '--dbname', default='wa',
+            help='The name of the database to create.')
+        self.parser.add_argument(
+            '-f', '--force', action='store_true',
+            help='Force overwrite the existing database if one exists.')
+        self.parser.add_argument(
+            '-F', '--force-update-config', action='store_true',
+            help='Force update the config file if an entry exists.')
+        self.parser.add_argument(
+            '-r', '--config-file', default=settings.user_config_file,
+            help='Path to the config file to be updated.')
+        self.parser.add_argument(
+            '-x', '--schema-version', action='store_true',
+            help='Display the current schema version.')
+
+    def execute(self, state, args):  # pylint: disable=too-many-branches
+        if not psycopg2:
+            raise CommandError(
+                'The module psycopg2 is required for the wa ' +
+                'create database command.')
+        self.get_schema(self.schemafilepath)
+
+        # Display the version if needed and exit
+        if args.schema_version:
+            self.logger.info(
+                'The current schema version is {}'.format(self.schemaversion))
+            return
+
+        if args.dbname == 'postgres':
+            raise ValueError('Databasename to create cannot be postgres.')
+
+        # Open user configuration
+        with open(args.config_file, 'r') as config_file:
+            config = yaml.load(config_file)
+            if 'postgres' in config and not args.force_update_config:
+                raise CommandError(
+                    "The entry 'postgres' already exists in the config file. " +
+                    "Please specify the -F flag to force an update.")
+
+        possible_connection_errors = [
+            (
+                re.compile('FATAL:  role ".*" does not exist'),
+                'Username does not exist or password is incorrect'
+            ),
+            (
+                re.compile('FATAL:  password authentication failed for user'),
+                'Password was incorrect'
+            ),
+            (
+                re.compile('fe_sendauth: no password supplied'),
+                'Passwordless connection is not enabled. '
+                'Please enable trust in pg_hba for this host '
+                'or use a password'
+            ),
+            (
+                re.compile('FATAL:  no pg_hba.conf entry for'),
+                'Host is not allowed to connect to the specified database '
+                'using this user according to pg_hba.conf. Please change the '
+                'rules in pg_hba or your connection method'
+            ),
+            (
+                re.compile('FATAL:  pg_hba.conf rejects connection'),
+                'Connection was rejected by pg_hba.conf'
+            ),
+        ]
+
+        def predicate(error, handle):
+            if handle[0].match(str(error)):
+                raise CommandError(handle[1] + ': \n' + str(error))
+
+        # Attempt to create database
+        try:
+            self.create_database(args)
+        except OperationalError as e:
+            for handle in possible_connection_errors:
+                predicate(e, handle)
+            raise e
+
+        # Update the configuration file
+        _update_configuration_file(args, config)
+
+    def create_database(self, args):
+        _check_database_existence(args)
+
+        _create_database_postgres(args)
+
+        _apply_database_schema(args, self.sql_commands, self.schema_major, self.schema_minor)
+
+        self.logger.debug(
+            "Successfully created the database {}".format(args.dbname))
+
+    def get_schema(self, schemafilepath):
+        postgres_output_processor_dir = os.path.dirname(__file__)
+        sqlfile = open(os.path.join(
+            postgres_output_processor_dir, schemafilepath))
+        self.sql_commands = sqlfile.read()
+        sqlfile.close()
+        # Extract schema version
+        if self.sql_commands.startswith('--!VERSION'):
+            splitcommands = self.sql_commands.split('!ENDVERSION!\n')
+            self.schemaversion = splitcommands[0].strip('--!VERSION!')
+            (self.schema_major, self.schema_minor) = self.schemaversion.split('.')
+            self.sql_commands = splitcommands[1]
+
+
 class CreateAgendaSubcommand(SubCommand):
 
     name = 'agenda'
@@ -181,6 +327,7 @@ class CreateCommand(ComplexCommand):
     object-specific arguments.
     '''
     subcmd_classes = [
+        CreateDatabaseSubcommand,
         CreateWorkloadSubcommand,
         CreateAgendaSubcommand,
         CreatePackageSubcommand,
@@ -280,3 +427,62 @@ def get_class_name(name, postfix=''):
 def touch(path):
     with open(path, 'w') as _: # NOQA
         pass
+
+
+def _check_database_existence(args):
+    try:
+        connect(dbname=args.dbname, user=args.username,
+                password=args.password, host=args.postgres_host, port=args.postgres_port)
+    except OperationalError as e:
+        # Expect an operational error (database's non-existence)
+        if not re.compile('FATAL:  database ".*" does not exist').match(str(e)):
+            raise e
+    else:
+        if not args.force:
+            raise CommandError(
+                "Database {} already exists. ".format(args.dbname) +
+                "Please specify the -f flag to create it from afresh."
+            )
+
+
+def _create_database_postgres(args):  # pylint: disable=no-self-use
+    conn = connect(dbname='postgres', user=args.username,
+                   password=args.password, host=args.postgres_host, port=args.postgres_port)
+    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    cursor = conn.cursor()
+    cursor.execute('DROP DATABASE IF EXISTS ' + args.dbname)
+    cursor.execute('CREATE DATABASE ' + args.dbname)
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+
+def _apply_database_schema(args, sql_commands, schema_major, schema_minor):
+    conn = connect(dbname=args.dbname, user=args.username,
+                   password=args.password, host=args.postgres_host, port=args.postgres_port)
+    cursor = conn.cursor()
+    cursor.execute(sql_commands)
+
+    extras.register_uuid()
+    cursor.execute("INSERT INTO DatabaseMeta VALUES (%s, %s, %s)",
+                   (
+                       uuid.uuid4(),
+                       schema_major,
+                       schema_minor
+                   )
+                   )
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+
+def _update_configuration_file(args, config):
+    ''' Update the user configuration file with the newly created database's
+        configuration.
+        '''
+    config['postgres'] = OrderedDict(
+        [('host', args.postgres_host), ('port', args.postgres_port),
+         ('dbname', args.dbname), ('username', args.username), ('password', args.password)])
+    with open(args.config_file, 'w+') as config_file:
+        yaml.dump(config, config_file)

wa/commands/postgres_schema.sql

@@ -0,0 +1,170 @@
+--!VERSION!1.1!ENDVERSION!
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+CREATE EXTENSION IF NOT EXISTS "lo";
+
+-- In future, it may be useful to implement rules on which Parameter oid fields can be none depeendent on the value in the type column;
+
+DROP TABLE IF EXISTS DatabaseMeta;
+DROP TABLE IF EXISTS Parameters;
+DROP TABLE IF EXISTS Classifiers;
+DROP TABLE IF EXISTS LargeObjects;
+DROP TABLE IF EXISTS Artifacts;
+DROP TABLE IF EXISTS Metrics;
+DROP TABLE IF EXISTS Augmentations;
+DROP TABLE IF EXISTS Jobs_Augs;
+DROP TABLE IF EXISTS ResourceGetters;
+DROP TABLE IF EXISTS Events;
+DROP TABLE IF EXISTS Targets;
+DROP TABLE IF EXISTS Jobs;
+DROP TABLE IF EXISTS Runs;
+
+DROP TYPE IF EXISTS status_enum;
+DROP TYPE IF EXISTS param_enum;
+
+CREATE TYPE status_enum AS ENUM ('UNKNOWN(0)','NEW(1)','PENDING(2)','STARTED(3)','CONNECTED(4)', 'INITIALIZED(5)', 'RUNNING(6)', 'OK(7)', 'PARTIAL(8)', 'FAILED(9)', 'ABORTED(10)', 'SKIPPED(11)');
+
+CREATE TYPE param_enum AS ENUM ('workload', 'resource_getter', 'augmentation', 'device', 'runtime', 'boot');
+
+-- In future, it might be useful to create an ENUM type for the artifact kind, or simply a generic enum type;
+
+CREATE TABLE DatabaseMeta (
+    oid uuid NOT NULL,
+    schema_major int,
+    schema_minor int,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Runs (
+    oid uuid NOT NULL,
+    event_summary text,
+    basepath text,
+    status status_enum,
+    timestamp timestamp,
+    run_name text,
+    project text,
+    retry_on_status status_enum[],
+    max_retries int,
+    bail_on_init_failure boolean,
+    allow_phone_home boolean,
+    run_uuid uuid,
+    start_time timestamp,
+    end_time timestamp,
+    metadata jsonb,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Jobs (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    status status_enum,
+    retries int,
+    label text,
+    job_id text,
+    iterations int,
+    workload_name text,
+    metadata jsonb,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Targets (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    target text,
+    cpus text[],
+    os text,
+    os_version jsonb,
+    hostid int,
+    hostname text,
+    abi text,
+    is_rooted boolean,
+    kernel_version text,
+    kernel_release text,
+    kernel_sha1 text,
+    kernel_config text[],
+    sched_features text[],
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Events (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    job_oid uuid references Jobs(oid),
+    timestamp timestamp,
+    message text,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE ResourceGetters (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    name text,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Augmentations (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    name text,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Jobs_Augs (
+    oid uuid NOT NULL,
+    job_oid uuid NOT NULL references Jobs(oid),
+    augmentation_oid uuid NOT NULL references Augmentations(oid),
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Metrics (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    job_oid uuid references Jobs(oid),
+    name text,
+    value double precision,
+    units text,
+    lower_is_better boolean,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE LargeObjects (
+    oid uuid NOT NULL,
+    lo_oid lo NOT NULL,
+    PRIMARY KEY (oid)
+);
+
+-- Trigger that allows you to manage large objects from the LO table directly;
+CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON LargeObjects
+    FOR EACH ROW EXECUTE PROCEDURE lo_manage(lo_oid);
+
+CREATE TABLE Artifacts (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    job_oid uuid references Jobs(oid),
+    name text,
+    large_object_uuid uuid NOT NULL references LargeObjects(oid),
+    description text,
+    kind text,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Classifiers (
+    oid uuid NOT NULL,
+    artifact_oid uuid references Artifacts(oid),
+    metric_oid uuid references Metrics(oid),
+    key text,
+    value text,
+    PRIMARY KEY (oid)
+);
+
+CREATE TABLE Parameters (
+    oid uuid NOT NULL,
+    run_oid uuid NOT NULL references Runs(oid),
+    job_oid uuid references Jobs(oid),
+    augmentation_oid uuid references Augmentations(oid),
+    resource_getter_oid uuid references ResourceGetters(oid),
+    name text,
+    value text,
+    value_type text,
+    type param_enum,
+    PRIMARY KEY (oid)
+);

wa/commands/schema_changelog.rst

@@ -0,0 +1,9 @@
+# 1
+## 1.0
+- First version
+## 1.1
+- LargeObjects table added as a substitute for the previous plan to
+  use the filesystem and a path reference to store artifacts. This
+  was done following an extended discussion and tests that verified
+  that the savings in processing power were not enough to warrant
+  the creation of a dedicated server or file handler.