mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-11-04 09:02:12 +00:00
Add WA Create Database Command
Add a command to create a PostgreSQL database with supplied parameters which draws its structure from the supplied schema (Version 1.1). This database is of a format intended to be used with the forthcoming WA Postgres output processor.
This commit is contained in:
Waleed El-Geresy
committed by
Marc Bonnici
Marc Bonnici
parent
ca03f21f46
commit
bb255de9ad
5
setup.py
Normal file → Executable file
5
setup.py
Normal file → Executable file
@@ -31,7 +31,7 @@ wa_dir = os.path.join(os.path.dirname(__file__), 'wa')
|
||||
sys.path.insert(0, os.path.join(wa_dir, 'framework'))
|
||||
from version import get_wa_version, get_wa_version_with_commit
|
||||
|
||||
# happends if falling back to distutils
|
||||
# happens if falling back to distutils
|
||||
warnings.filterwarnings('ignore', "Unknown distribution option: 'install_requires'")
|
||||
warnings.filterwarnings('ignore', "Unknown distribution option: 'extras_require'")
|
||||
|
||||
@@ -41,7 +41,7 @@ except OSError:
|
||||
pass
|
||||
|
||||
packages = []
|
||||
data_files = {}
|
||||
data_files = {'': [os.path.join(wa_dir, 'commands', 'postgres_schema.sql')]}
|
||||
source_dir = os.path.dirname(__file__)
|
||||
for root, dirs, files in os.walk(wa_dir):
|
||||
rel_dir = os.path.relpath(root, source_dir)
|
||||
@@ -67,6 +67,7 @@ params = dict(
|
||||
version=get_wa_version_with_commit(),
|
||||
packages=packages,
|
||||
package_data=data_files,
|
||||
include_package_data=True,
|
||||
scripts=scripts,
|
||||
url='https://github.com/ARM-software/workload-automation',
|
||||
license='Apache v2',
|
||||
|
||||
@@ -13,16 +13,26 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
import stat
|
||||
import shutil
|
||||
import string
|
||||
import re
|
||||
import uuid
|
||||
import getpass
|
||||
from collections import OrderedDict
|
||||
from distutils.dir_util import copy_tree # pylint: disable=no-name-in-module, import-error
|
||||
|
||||
from devlib.utils.types import identifier
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2 import connect, OperationalError, extras
|
||||
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
|
||||
except ImportError as e:
|
||||
psycopg2 = None
|
||||
import_error_msg = e.args[0] if e.args else str(e)
|
||||
|
||||
from wa import ComplexCommand, SubCommand, pluginloader, settings
|
||||
from wa.framework.target.descriptor import list_target_descriptions
|
||||
@@ -36,6 +46,142 @@ from wa.utils.serializer import yaml
|
||||
TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'templates')
|
||||
|
||||
|
||||
class CreateDatabaseSubcommand(SubCommand):
|
||||
|
||||
name = 'database'
|
||||
description = """
|
||||
Create a Postgresql database which is compatible with the WA Postgres
|
||||
output processor.
|
||||
"""
|
||||
|
||||
schemafilepath = 'postgres_schema.sql'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(CreateDatabaseSubcommand, self).__init__(*args, **kwargs)
|
||||
self.sql_commands = None
|
||||
self.schemaversion = None
|
||||
self.schema_major = None
|
||||
self.schema_minor = None
|
||||
|
||||
def initialize(self, context):
|
||||
self.parser.add_argument(
|
||||
'-a', '--postgres-host', default='localhost',
|
||||
help='The host on which to create the database.')
|
||||
self.parser.add_argument(
|
||||
'-k', '--postgres-port', default='5432',
|
||||
help='The port on which the PostgreSQL server is running.')
|
||||
self.parser.add_argument(
|
||||
'-u', '--username', default='postgres',
|
||||
help='The username with which to connect to the server.')
|
||||
self.parser.add_argument(
|
||||
'-p', '--password',
|
||||
help='The password for the user account.')
|
||||
self.parser.add_argument(
|
||||
'-d', '--dbname', default='wa',
|
||||
help='The name of the database to create.')
|
||||
self.parser.add_argument(
|
||||
'-f', '--force', action='store_true',
|
||||
help='Force overwrite the existing database if one exists.')
|
||||
self.parser.add_argument(
|
||||
'-F', '--force-update-config', action='store_true',
|
||||
help='Force update the config file if an entry exists.')
|
||||
self.parser.add_argument(
|
||||
'-r', '--config-file', default=settings.user_config_file,
|
||||
help='Path to the config file to be updated.')
|
||||
self.parser.add_argument(
|
||||
'-x', '--schema-version', action='store_true',
|
||||
help='Display the current schema version.')
|
||||
|
||||
def execute(self, state, args): # pylint: disable=too-many-branches
|
||||
if not psycopg2:
|
||||
raise CommandError(
|
||||
'The module psycopg2 is required for the wa ' +
|
||||
'create database command.')
|
||||
self.get_schema(self.schemafilepath)
|
||||
|
||||
# Display the version if needed and exit
|
||||
if args.schema_version:
|
||||
self.logger.info(
|
||||
'The current schema version is {}'.format(self.schemaversion))
|
||||
return
|
||||
|
||||
if args.dbname == 'postgres':
|
||||
raise ValueError('Databasename to create cannot be postgres.')
|
||||
|
||||
# Open user configuration
|
||||
with open(args.config_file, 'r') as config_file:
|
||||
config = yaml.load(config_file)
|
||||
if 'postgres' in config and not args.force_update_config:
|
||||
raise CommandError(
|
||||
"The entry 'postgres' already exists in the config file. " +
|
||||
"Please specify the -F flag to force an update.")
|
||||
|
||||
possible_connection_errors = [
|
||||
(
|
||||
re.compile('FATAL: role ".*" does not exist'),
|
||||
'Username does not exist or password is incorrect'
|
||||
),
|
||||
(
|
||||
re.compile('FATAL: password authentication failed for user'),
|
||||
'Password was incorrect'
|
||||
),
|
||||
(
|
||||
re.compile('fe_sendauth: no password supplied'),
|
||||
'Passwordless connection is not enabled. '
|
||||
'Please enable trust in pg_hba for this host '
|
||||
'or use a password'
|
||||
),
|
||||
(
|
||||
re.compile('FATAL: no pg_hba.conf entry for'),
|
||||
'Host is not allowed to connect to the specified database '
|
||||
'using this user according to pg_hba.conf. Please change the '
|
||||
'rules in pg_hba or your connection method'
|
||||
),
|
||||
(
|
||||
re.compile('FATAL: pg_hba.conf rejects connection'),
|
||||
'Connection was rejected by pg_hba.conf'
|
||||
),
|
||||
]
|
||||
|
||||
def predicate(error, handle):
|
||||
if handle[0].match(str(error)):
|
||||
raise CommandError(handle[1] + ': \n' + str(error))
|
||||
|
||||
# Attempt to create database
|
||||
try:
|
||||
self.create_database(args)
|
||||
except OperationalError as e:
|
||||
for handle in possible_connection_errors:
|
||||
predicate(e, handle)
|
||||
raise e
|
||||
|
||||
# Update the configuration file
|
||||
_update_configuration_file(args, config)
|
||||
|
||||
def create_database(self, args):
|
||||
_check_database_existence(args)
|
||||
|
||||
_create_database_postgres(args)
|
||||
|
||||
_apply_database_schema(args, self.sql_commands, self.schema_major, self.schema_minor)
|
||||
|
||||
self.logger.debug(
|
||||
"Successfully created the database {}".format(args.dbname))
|
||||
|
||||
def get_schema(self, schemafilepath):
|
||||
postgres_output_processor_dir = os.path.dirname(__file__)
|
||||
sqlfile = open(os.path.join(
|
||||
postgres_output_processor_dir, schemafilepath))
|
||||
self.sql_commands = sqlfile.read()
|
||||
sqlfile.close()
|
||||
# Extract schema version
|
||||
if self.sql_commands.startswith('--!VERSION'):
|
||||
splitcommands = self.sql_commands.split('!ENDVERSION!\n')
|
||||
self.schemaversion = splitcommands[0].strip('--!VERSION!')
|
||||
(self.schema_major, self.schema_minor) = self.schemaversion.split('.')
|
||||
self.sql_commands = splitcommands[1]
|
||||
|
||||
|
||||
class CreateAgendaSubcommand(SubCommand):
|
||||
|
||||
name = 'agenda'
|
||||
@@ -181,6 +327,7 @@ class CreateCommand(ComplexCommand):
|
||||
object-specific arguments.
|
||||
'''
|
||||
subcmd_classes = [
|
||||
CreateDatabaseSubcommand,
|
||||
CreateWorkloadSubcommand,
|
||||
CreateAgendaSubcommand,
|
||||
CreatePackageSubcommand,
|
||||
@@ -280,3 +427,62 @@ def get_class_name(name, postfix=''):
|
||||
def touch(path):
|
||||
with open(path, 'w') as _: # NOQA
|
||||
pass
|
||||
|
||||
|
||||
def _check_database_existence(args):
|
||||
try:
|
||||
connect(dbname=args.dbname, user=args.username,
|
||||
password=args.password, host=args.postgres_host, port=args.postgres_port)
|
||||
except OperationalError as e:
|
||||
# Expect an operational error (database's non-existence)
|
||||
if not re.compile('FATAL: database ".*" does not exist').match(str(e)):
|
||||
raise e
|
||||
else:
|
||||
if not args.force:
|
||||
raise CommandError(
|
||||
"Database {} already exists. ".format(args.dbname) +
|
||||
"Please specify the -f flag to create it from afresh."
|
||||
)
|
||||
|
||||
|
||||
def _create_database_postgres(args): # pylint: disable=no-self-use
|
||||
conn = connect(dbname='postgres', user=args.username,
|
||||
password=args.password, host=args.postgres_host, port=args.postgres_port)
|
||||
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DROP DATABASE IF EXISTS ' + args.dbname)
|
||||
cursor.execute('CREATE DATABASE ' + args.dbname)
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
def _apply_database_schema(args, sql_commands, schema_major, schema_minor):
|
||||
conn = connect(dbname=args.dbname, user=args.username,
|
||||
password=args.password, host=args.postgres_host, port=args.postgres_port)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(sql_commands)
|
||||
|
||||
extras.register_uuid()
|
||||
cursor.execute("INSERT INTO DatabaseMeta VALUES (%s, %s, %s)",
|
||||
(
|
||||
uuid.uuid4(),
|
||||
schema_major,
|
||||
schema_minor
|
||||
)
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
def _update_configuration_file(args, config):
|
||||
''' Update the user configuration file with the newly created database's
|
||||
configuration.
|
||||
'''
|
||||
config['postgres'] = OrderedDict(
|
||||
[('host', args.postgres_host), ('port', args.postgres_port),
|
||||
('dbname', args.dbname), ('username', args.username), ('password', args.password)])
|
||||
with open(args.config_file, 'w+') as config_file:
|
||||
yaml.dump(config, config_file)
|
||||
|
||||
170
wa/commands/postgres_schema.sql
Normal file
170
wa/commands/postgres_schema.sql
Normal file
@@ -0,0 +1,170 @@
|
||||
--!VERSION!1.1!ENDVERSION!
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "lo";
|
||||
|
||||
-- In future, it may be useful to implement rules on which Parameter oid fields can be none depeendent on the value in the type column;
|
||||
|
||||
DROP TABLE IF EXISTS DatabaseMeta;
|
||||
DROP TABLE IF EXISTS Parameters;
|
||||
DROP TABLE IF EXISTS Classifiers;
|
||||
DROP TABLE IF EXISTS LargeObjects;
|
||||
DROP TABLE IF EXISTS Artifacts;
|
||||
DROP TABLE IF EXISTS Metrics;
|
||||
DROP TABLE IF EXISTS Augmentations;
|
||||
DROP TABLE IF EXISTS Jobs_Augs;
|
||||
DROP TABLE IF EXISTS ResourceGetters;
|
||||
DROP TABLE IF EXISTS Events;
|
||||
DROP TABLE IF EXISTS Targets;
|
||||
DROP TABLE IF EXISTS Jobs;
|
||||
DROP TABLE IF EXISTS Runs;
|
||||
|
||||
DROP TYPE IF EXISTS status_enum;
|
||||
DROP TYPE IF EXISTS param_enum;
|
||||
|
||||
CREATE TYPE status_enum AS ENUM ('UNKNOWN(0)','NEW(1)','PENDING(2)','STARTED(3)','CONNECTED(4)', 'INITIALIZED(5)', 'RUNNING(6)', 'OK(7)', 'PARTIAL(8)', 'FAILED(9)', 'ABORTED(10)', 'SKIPPED(11)');
|
||||
|
||||
CREATE TYPE param_enum AS ENUM ('workload', 'resource_getter', 'augmentation', 'device', 'runtime', 'boot');
|
||||
|
||||
-- In future, it might be useful to create an ENUM type for the artifact kind, or simply a generic enum type;
|
||||
|
||||
CREATE TABLE DatabaseMeta (
|
||||
oid uuid NOT NULL,
|
||||
schema_major int,
|
||||
schema_minor int,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Runs (
|
||||
oid uuid NOT NULL,
|
||||
event_summary text,
|
||||
basepath text,
|
||||
status status_enum,
|
||||
timestamp timestamp,
|
||||
run_name text,
|
||||
project text,
|
||||
retry_on_status status_enum[],
|
||||
max_retries int,
|
||||
bail_on_init_failure boolean,
|
||||
allow_phone_home boolean,
|
||||
run_uuid uuid,
|
||||
start_time timestamp,
|
||||
end_time timestamp,
|
||||
metadata jsonb,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Jobs (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
status status_enum,
|
||||
retries int,
|
||||
label text,
|
||||
job_id text,
|
||||
iterations int,
|
||||
workload_name text,
|
||||
metadata jsonb,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Targets (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
target text,
|
||||
cpus text[],
|
||||
os text,
|
||||
os_version jsonb,
|
||||
hostid int,
|
||||
hostname text,
|
||||
abi text,
|
||||
is_rooted boolean,
|
||||
kernel_version text,
|
||||
kernel_release text,
|
||||
kernel_sha1 text,
|
||||
kernel_config text[],
|
||||
sched_features text[],
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Events (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
job_oid uuid references Jobs(oid),
|
||||
timestamp timestamp,
|
||||
message text,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE ResourceGetters (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
name text,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Augmentations (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
name text,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Jobs_Augs (
|
||||
oid uuid NOT NULL,
|
||||
job_oid uuid NOT NULL references Jobs(oid),
|
||||
augmentation_oid uuid NOT NULL references Augmentations(oid),
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Metrics (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
job_oid uuid references Jobs(oid),
|
||||
name text,
|
||||
value double precision,
|
||||
units text,
|
||||
lower_is_better boolean,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE LargeObjects (
|
||||
oid uuid NOT NULL,
|
||||
lo_oid lo NOT NULL,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
-- Trigger that allows you to manage large objects from the LO table directly;
|
||||
CREATE TRIGGER t_raster BEFORE UPDATE OR DELETE ON LargeObjects
|
||||
FOR EACH ROW EXECUTE PROCEDURE lo_manage(lo_oid);
|
||||
|
||||
CREATE TABLE Artifacts (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
job_oid uuid references Jobs(oid),
|
||||
name text,
|
||||
large_object_uuid uuid NOT NULL references LargeObjects(oid),
|
||||
description text,
|
||||
kind text,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Classifiers (
|
||||
oid uuid NOT NULL,
|
||||
artifact_oid uuid references Artifacts(oid),
|
||||
metric_oid uuid references Metrics(oid),
|
||||
key text,
|
||||
value text,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
|
||||
CREATE TABLE Parameters (
|
||||
oid uuid NOT NULL,
|
||||
run_oid uuid NOT NULL references Runs(oid),
|
||||
job_oid uuid references Jobs(oid),
|
||||
augmentation_oid uuid references Augmentations(oid),
|
||||
resource_getter_oid uuid references ResourceGetters(oid),
|
||||
name text,
|
||||
value text,
|
||||
value_type text,
|
||||
type param_enum,
|
||||
PRIMARY KEY (oid)
|
||||
);
|
||||
9
wa/commands/schema_changelog.rst
Normal file
9
wa/commands/schema_changelog.rst
Normal file
@@ -0,0 +1,9 @@
|
||||
# 1
|
||||
## 1.0
|
||||
- First version
|
||||
## 1.1
|
||||
- LargeObjects table added as a substitute for the previous plan to
|
||||
use the filesystem and a path reference to store artifacts. This
|
||||
was done following an extended discussion and tests that verified
|
||||
that the savings in processing power were not enough to warrant
|
||||
the creation of a dedicated server or file handler.
|
||||
Reference in New Issue
Block a user