diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 38635605..7790de26 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ wa_dir = os.path.join(os.path.dirname(__file__), 'wa') sys.path.insert(0, os.path.join(wa_dir, 'framework')) from version import get_wa_version, get_wa_version_with_commit -# happends if falling back to distutils +# happens if falling back to distutils warnings.filterwarnings('ignore', "Unknown distribution option: 'install_requires'") warnings.filterwarnings('ignore', "Unknown distribution option: 'extras_require'") @@ -41,7 +41,7 @@ except OSError: pass packages = [] -data_files = {} +data_files = {'': [os.path.join(wa_dir, 'commands', 'postgres_schema.sql')]} source_dir = os.path.dirname(__file__) for root, dirs, files in os.walk(wa_dir): rel_dir = os.path.relpath(root, source_dir) @@ -67,6 +67,7 @@ params = dict( version=get_wa_version_with_commit(), packages=packages, package_data=data_files, + include_package_data=True, scripts=scripts, url='https://github.com/ARM-software/workload-automation', license='Apache v2', diff --git a/wa/commands/create.py b/wa/commands/create.py index 4b18ec35..dad3ebe5 100644 --- a/wa/commands/create.py +++ b/wa/commands/create.py @@ -13,16 +13,26 @@ # limitations under the License. # + import os import sys import stat import shutil import string +import re +import uuid import getpass from collections import OrderedDict from distutils.dir_util import copy_tree # pylint: disable=no-name-in-module, import-error from devlib.utils.types import identifier +try: + import psycopg2 + from psycopg2 import connect, OperationalError, extras + from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +except ImportError as e: + psycopg2 = None + import_error_msg = e.args[0] if e.args else str(e) from wa import ComplexCommand, SubCommand, pluginloader, settings from wa.framework.target.descriptor import list_target_descriptions @@ -36,6 +46,142 @@ from wa.utils.serializer import yaml TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates') +class CreateDatabaseSubcommand(SubCommand): + + name = 'database' + description = """ + Create a Postgresql database which is compatible with the WA Postgres + output processor. + """ + + schemafilepath = 'postgres_schema.sql' + + def __init__(self, *args, **kwargs): + super(CreateDatabaseSubcommand, self).__init__(*args, **kwargs) + self.sql_commands = None + self.schemaversion = None + self.schema_major = None + self.schema_minor = None + + def initialize(self, context): + self.parser.add_argument( + '-a', '--postgres-host', default='localhost', + help='The host on which to create the database.') + self.parser.add_argument( + '-k', '--postgres-port', default='5432', + help='The port on which the PostgreSQL server is running.') + self.parser.add_argument( + '-u', '--username', default='postgres', + help='The username with which to connect to the server.') + self.parser.add_argument( + '-p', '--password', + help='The password for the user account.') + self.parser.add_argument( + '-d', '--dbname', default='wa', + help='The name of the database to create.') + self.parser.add_argument( + '-f', '--force', action='store_true', + help='Force overwrite the existing database if one exists.') + self.parser.add_argument( + '-F', '--force-update-config', action='store_true', + help='Force update the config file if an entry exists.') + self.parser.add_argument( + '-r', '--config-file', default=settings.user_config_file, + help='Path to the config file to be updated.') + self.parser.add_argument( + '-x', '--schema-version', action='store_true', + help='Display the current schema version.') + + def execute(self, state, args): # pylint: disable=too-many-branches + if not psycopg2: + raise CommandError( + 'The module psycopg2 is required for the wa ' + + 'create database command.') + self.get_schema(self.schemafilepath) + + # Display the version if needed and exit + if args.schema_version: + self.logger.info( + 'The current schema version is {}'.format(self.schemaversion)) + return + + if args.dbname == 'postgres': + raise ValueError('Databasename to create cannot be postgres.') + + # Open user configuration + with open(args.config_file, 'r') as config_file: + config = yaml.load(config_file) + if 'postgres' in config and not args.force_update_config: + raise CommandError( + "The entry 'postgres' already exists in the config file. " + + "Please specify the -F flag to force an update.") + + possible_connection_errors = [ + ( + re.compile('FATAL: role ".*" does not exist'), + 'Username does not exist or password is incorrect' + ), + ( + re.compile('FATAL: password authentication failed for user'), + 'Password was incorrect' + ), + ( + re.compile('fe_sendauth: no password supplied'), + 'Passwordless connection is not enabled. ' + 'Please enable trust in pg_hba for this host ' + 'or use a password' + ), + ( + re.compile('FATAL: no pg_hba.conf entry for'), + 'Host is not allowed to connect to the specified database ' + 'using this user according to pg_hba.conf. Please change the ' + 'rules in pg_hba or your connection method' + ), + ( + re.compile('FATAL: pg_hba.conf rejects connection'), + 'Connection was rejected by pg_hba.conf' + ), + ] + + def predicate(error, handle): + if handle[0].match(str(error)): + raise CommandError(handle[1] + ': \n' + str(error)) + + # Attempt to create database + try: + self.create_database(args) + except OperationalError as e: + for handle in possible_connection_errors: + predicate(e, handle) + raise e + + # Update the configuration file + _update_configuration_file(args, config) + + def create_database(self, args): + _check_database_existence(args) + + _create_database_postgres(args) + + _apply_database_schema(args, self.sql_commands, self.schema_major, self.schema_minor) + + self.logger.debug( + "Successfully created the database {}".format(args.dbname)) + + def get_schema(self, schemafilepath): + postgres_output_processor_dir = os.path.dirname(__file__) + sqlfile = open(os.path.join( + postgres_output_processor_dir, schemafilepath)) + self.sql_commands = sqlfile.read() + sqlfile.close() + # Extract schema version + if self.sql_commands.startswith('--!VERSION'): + splitcommands = self.sql_commands.split('!ENDVERSION!\n') + self.schemaversion = splitcommands[0].strip('--!VERSION!') + (self.schema_major, self.schema_minor) = self.schemaversion.split('.') + self.sql_commands = splitcommands[1] + + class CreateAgendaSubcommand(SubCommand): name = 'agenda' @@ -181,6 +327,7 @@ class CreateCommand(ComplexCommand): object-specific arguments. ''' subcmd_classes = [ + CreateDatabaseSubcommand, CreateWorkloadSubcommand, CreateAgendaSubcommand, CreatePackageSubcommand, @@ -280,3 +427,62 @@ def get_class_name(name, postfix=''): def touch(path): with open(path, 'w') as _: # NOQA pass + + +def _check_database_existence(args): + try: + connect(dbname=args.dbname, user=args.username, + password=args.password, host=args.postgres_host, port=args.postgres_port) + except OperationalError as e: + # Expect an operational error (database's non-existence) + if not re.compile('FATAL: database ".*" does not exist').match(str(e)): + raise e + else: + if not args.force: + raise CommandError( + "Database {} already exists. ".format(args.dbname) + + "Please specify the -f flag to create it from afresh." + ) + + +def _create_database_postgres(args): # pylint: disable=no-self-use + conn = connect(dbname='postgres', user=args.username, + password=args.password, host=args.postgres_host, port=args.postgres_port) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + cursor = conn.cursor() + cursor.execute('DROP DATABASE IF EXISTS ' + args.dbname) + cursor.execute('CREATE DATABASE ' + args.dbname) + conn.commit() + cursor.close() + conn.close() + + +def _apply_database_schema(args, sql_commands, schema_major, schema_minor): + conn = connect(dbname=args.dbname, user=args.username, + password=args.password, host=args.postgres_host, port=args.postgres_port) + cursor = conn.cursor() + cursor.execute(sql_commands) + + extras.register_uuid() + cursor.execute("INSERT INTO DatabaseMeta VALUES (%s, %s, %s)", + ( + uuid.uuid4(), + schema_major, + schema_minor + ) + ) + + conn.commit() + cursor.close() + conn.close() + + +def _update_configuration_file(args, config): + ''' Update the user configuration file with the newly created database's + configuration. + ''' + config['postgres'] = OrderedDict( + [('host', args.postgres_host), ('port', args.postgres_port), + ('dbname', args.dbname), ('username', args.username), ('password', args.password)]) + with open(args.config_file, 'w+') as config_file: + yaml.dump(config, config_file) diff --git a/wa/commands/postgres_schema.sql b/wa/commands/postgres_schema.sql new file mode 100644 index 00000000..26bca17a --- /dev/null +++ b/wa/commands/postgres_schema.sql @@ -0,0 +1,170 @@ +--!VERSION!1.1!ENDVERSION! +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS "lo"; + +-- In future, it may be useful to implement rules on which Parameter oid fields can be none depeendent on the value in the type column; + +DROP TABLE IF EXISTS DatabaseMeta; +DROP TABLE IF EXISTS Parameters; +DROP TABLE IF EXISTS Classifiers; +DROP TABLE IF EXISTS LargeObjects; +DROP TABLE IF EXISTS Artifacts; +DROP TABLE IF EXISTS Metrics; +DROP TABLE IF EXISTS Augmentations; +DROP TABLE IF EXISTS Jobs_Augs; +DROP TABLE IF EXISTS ResourceGetters; +DROP TABLE IF EXISTS Events; +DROP TABLE IF EXISTS Targets; +DROP TABLE IF EXISTS Jobs; +DROP TABLE IF EXISTS Runs; + +DROP TYPE IF EXISTS status_enum; +DROP TYPE IF EXISTS param_enum; + +CREATE TYPE status_enum AS ENUM ('UNKNOWN(0)','NEW(1)','PENDING(2)','STARTED(3)','CONNECTED(4)', 'INITIALIZED(5)', 'RUNNING(6)', 'OK(7)', 'PARTIAL(8)', 'FAILED(9)', 'ABORTED(10)', 'SKIPPED(11)'); + +CREATE TYPE param_enum AS ENUM ('workload', 'resource_getter', 'augmentation', 'device', 'runtime', 'boot'); + +-- In future, it might be useful to create an ENUM type for the artifact kind, or simply a generic enum type; + +CREATE TABLE DatabaseMeta ( + oid uuid NOT NULL, + schema_major int, + schema_minor int, + PRIMARY KEY (oid) +); + +CREATE TABLE Runs ( + oid uuid NOT NULL, + event_summary text, + basepath text, + status status_enum, + timestamp timestamp, + run_name text, + project text, + retry_on_status status_enum[], + max_retries int, + bail_on_init_failure boolean, + allow_phone_home boolean, + run_uuid uuid, + start_time timestamp, + end_time timestamp, + metadata jsonb, + PRIMARY KEY (oid) +); + +CREATE TABLE Jobs ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + status status_enum, + retries int, + label text, + job_id text, + iterations int, + workload_name text, + metadata jsonb, + PRIMARY KEY (oid) +); + +CREATE TABLE Targets ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + target text, + cpus text[], + os text, + os_version jsonb, + hostid int, + hostname text, + abi text, + is_rooted boolean, + kernel_version text, + kernel_release text, + kernel_sha1 text, + kernel_config text[], + sched_features text[], + PRIMARY KEY (oid) +); + +CREATE TABLE Events ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + job_oid uuid references Jobs(oid), + timestamp timestamp, + message text, + PRIMARY KEY (oid) +); + +CREATE TABLE ResourceGetters ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + name text, + PRIMARY KEY (oid) +); + +CREATE TABLE Augmentations ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + name text, + PRIMARY KEY (oid) +); + +CREATE TABLE Jobs_Augs ( + oid uuid NOT NULL, + job_oid uuid NOT NULL references Jobs(oid), + augmentation_oid uuid NOT NULL references Augmentations(oid), + PRIMARY KEY (oid) +); + +CREATE TABLE Metrics ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + job_oid uuid references Jobs(oid), + name text, + value double precision, + units text, + lower_is_better boolean, + PRIMARY KEY (oid) +); + +CREATE TABLE LargeObjects ( + oid uuid NOT NULL, + lo_oid lo NOT NULL, + PRIMARY KEY (oid) +); + +-- Trigger that allows you to manage large objects from the LO table directly; +CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON LargeObjects + FOR EACH ROW EXECUTE PROCEDURE lo_manage(lo_oid); + +CREATE TABLE Artifacts ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + job_oid uuid references Jobs(oid), + name text, + large_object_uuid uuid NOT NULL references LargeObjects(oid), + description text, + kind text, + PRIMARY KEY (oid) +); + +CREATE TABLE Classifiers ( + oid uuid NOT NULL, + artifact_oid uuid references Artifacts(oid), + metric_oid uuid references Metrics(oid), + key text, + value text, + PRIMARY KEY (oid) +); + +CREATE TABLE Parameters ( + oid uuid NOT NULL, + run_oid uuid NOT NULL references Runs(oid), + job_oid uuid references Jobs(oid), + augmentation_oid uuid references Augmentations(oid), + resource_getter_oid uuid references ResourceGetters(oid), + name text, + value text, + value_type text, + type param_enum, + PRIMARY KEY (oid) +); diff --git a/wa/commands/schema_changelog.rst b/wa/commands/schema_changelog.rst new file mode 100644 index 00000000..862f9da4 --- /dev/null +++ b/wa/commands/schema_changelog.rst @@ -0,0 +1,9 @@ +# 1 +## 1.0 +- First version +## 1.1 +- LargeObjects table added as a substitute for the previous plan to + use the filesystem and a path reference to store artifacts. This + was done following an extended discussion and tests that verified + that the savings in processing power were not enough to warrant + the creation of a dedicated server or file handler.