1
0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-09-22 21:22:37 +01:00

Initial commit of open source Workload Automation.

This commit is contained in:
Sergei Trofimov
2015-03-10 13:09:31 +00:00
commit a747ec7e4c
412 changed files with 41401 additions and 0 deletions

View File

0
wlauto/external/daq_server/src/README vendored Normal file
View File

25
wlauto/external/daq_server/src/build.sh vendored Executable file
View File

@@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
python setup.py sdist
rm -rf build
rm -f MANIFEST
if [[ -d dist ]]; then
mv dist/*.tar.gz ..
rm -rf dist
fi
find . -iname \*.pyc -delete

View File

@@ -0,0 +1,17 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = '1.0.1'

View File

@@ -0,0 +1,380 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=E1101,E1103
import os
import sys
from twisted.internet import reactor
from twisted.internet.protocol import Protocol, ClientFactory, ReconnectingClientFactory
from twisted.internet.error import ConnectionLost, ConnectionDone
from twisted.protocols.basic import LineReceiver
if __name__ == '__main__': # for debugging
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from daqpower import log
from daqpower.common import DaqServerRequest, DaqServerResponse, Status
from daqpower.config import get_config_parser
__all__ = ['execute_command', 'run_send_command', 'Status']
class Command(object):
def __init__(self, name, **params):
self.name = name
self.params = params
class CommandResult(object):
def __init__(self):
self.status = None
self.message = None
self.data = None
def __str__(self):
return '{} {}'.format(self.status, self.message)
class CommandExecutorProtocol(Protocol):
def __init__(self, command, timeout=10, retries=1):
self.command = command
self.sent_request = None
self.waiting_for_response = False
self.keep_going = None
self.ports_to_pull = None
self.factory = None
self.timeoutCallback = None
self.timeout = timeout
self.retries = retries
self.retry_count = 0
def connectionMade(self):
if self.command.name == 'get_data':
self.sendRequest('list_port_files')
else:
self.sendRequest(self.command.name, **self.command.params)
def connectionLost(self, reason=ConnectionDone):
if isinstance(reason, ConnectionLost):
self.errorOut('connection lost: {}'.format(reason))
elif self.waiting_for_response:
self.errorOut('Server closed connection without sending a response.')
else:
log.debug('connection terminated.')
def sendRequest(self, command, **params):
self.sent_request = DaqServerRequest(command, params)
request_string = self.sent_request.serialize()
log.debug('sending request: {}'.format(request_string))
self.transport.write(''.join([request_string, '\r\n']))
self.timeoutCallback = reactor.callLater(self.timeout, self.requestTimedOut)
self.waiting_for_response = True
def dataReceived(self, data):
self.keep_going = False
if self.waiting_for_response:
self.waiting_for_response = False
self.timeoutCallback.cancel()
try:
response = DaqServerResponse.deserialize(data)
except Exception, e: # pylint: disable=W0703
self.errorOut('Invalid response: {} ({})'.format(data, e))
else:
if response.status != Status.ERROR:
self.processResponse(response) # may set self.keep_going
if not self.keep_going:
self.commandCompleted(response.status, response.message, response.data)
else:
self.errorOut(response.message)
else:
self.errorOut('unexpected data received: {}\n'.format(data))
def processResponse(self, response):
if self.sent_request.command in ['list_ports', 'list_port_files']:
self.processPortsResponse(response)
elif self.sent_request.command == 'list_devices':
self.processDevicesResponse(response)
elif self.sent_request.command == 'pull':
self.processPullResponse(response)
def processPortsResponse(self, response):
if 'ports' not in response.data:
self.errorOut('Response did not containt ports data: {} ({}).'.format(response, response.data))
ports = response.data['ports']
response.data = ports
if self.command.name == 'get_data':
if ports:
self.ports_to_pull = ports
self.sendPullRequest(self.ports_to_pull.pop())
else:
response.status = Status.OKISH
response.message = 'No ports were returned.'
def processDevicesResponse(self, response):
if 'devices' not in response.data:
self.errorOut('Response did not containt devices data: {} ({}).'.format(response, response.data))
ports = response.data['devices']
response.data = ports
def sendPullRequest(self, port_id):
self.sendRequest('pull', port_id=port_id)
self.keep_going = True
def processPullResponse(self, response):
if 'port_number' not in response.data:
self.errorOut('Response does not contain port number: {} ({}).'.format(response, response.data))
port_number = response.data.pop('port_number')
filename = self.sent_request.params['port_id'] + '.csv'
self.factory.initiateFileTransfer(filename, port_number)
if self.ports_to_pull:
self.sendPullRequest(self.ports_to_pull.pop())
def commandCompleted(self, status, message=None, data=None):
self.factory.result.status = status
self.factory.result.message = message
self.factory.result.data = data
self.transport.loseConnection()
def requestTimedOut(self):
self.retry_count += 1
if self.retry_count > self.retries:
self.errorOut("Request timed out; server failed to respond.")
else:
log.debug('Retrying...')
self.connectionMade()
def errorOut(self, message):
self.factory.errorOut(message)
class CommandExecutorFactory(ClientFactory):
protocol = CommandExecutorProtocol
wait_delay = 1
def __init__(self, config, command, timeout=10, retries=1):
self.config = config
self.command = command
self.timeout = timeout
self.retries = retries
self.result = CommandResult()
self.done = False
self.transfers_in_progress = {}
if command.name == 'get_data':
if 'output_directory' not in command.params:
self.errorOut('output_directory not specifed for get_data command.')
self.output_directory = command.params['output_directory']
if not os.path.isdir(self.output_directory):
log.debug('Creating output directory {}'.format(self.output_directory))
os.makedirs(self.output_directory)
def buildProtocol(self, addr):
protocol = CommandExecutorProtocol(self.command, self.timeout, self.retries)
protocol.factory = self
return protocol
def initiateFileTransfer(self, filename, port):
log.debug('Downloading {} from port {}'.format(filename, port))
filepath = os.path.join(self.output_directory, filename)
session = FileReceiverFactory(filepath, self)
connector = reactor.connectTCP(self.config.host, port, session)
self.transfers_in_progress[session] = connector
def transferComplete(self, session):
connector = self.transfers_in_progress[session]
log.debug('Transfer on port {} complete.'.format(connector.port))
del self.transfers_in_progress[session]
def clientConnectionLost(self, connector, reason):
if self.transfers_in_progress:
log.debug('Waiting for the transfer(s) to complete.')
self.waitForTransfersToCompleteAndExit()
def clientConnectionFailed(self, connector, reason):
self.result.status = Status.ERROR
self.result.message = 'Could not connect to server.'
self.waitForTransfersToCompleteAndExit()
def waitForTransfersToCompleteAndExit(self):
if self.transfers_in_progress:
reactor.callLater(self.wait_delay, self.waitForTransfersToCompleteAndExit)
else:
log.debug('Stopping the reactor.')
reactor.stop()
def errorOut(self, message):
self.result.status = Status.ERROR
self.result.message = message
reactor.crash()
def __str__(self):
return '<CommandExecutorProtocol {}>'.format(self.command.name)
__repr__ = __str__
class FileReceiver(LineReceiver): # pylint: disable=W0223
def __init__(self, path):
self.path = path
self.fh = None
self.factory = None
def connectionMade(self):
if os.path.isfile(self.path):
log.warning('overriding existing file.')
os.remove(self.path)
self.fh = open(self.path, 'w')
def connectionLost(self, reason=ConnectionDone):
if self.fh:
self.fh.close()
def lineReceived(self, line):
line = line.rstrip('\r\n') + '\n'
self.fh.write(line)
class FileReceiverFactory(ReconnectingClientFactory):
def __init__(self, path, owner):
self.path = path
self.owner = owner
def buildProtocol(self, addr):
protocol = FileReceiver(self.path)
protocol.factory = self
self.resetDelay()
return protocol
def clientConnectionLost(self, conector, reason):
if isinstance(reason, ConnectionLost):
log.error('Connection lost: {}'.format(reason))
ReconnectingClientFactory.clientConnectionLost(self, conector, reason)
else:
self.owner.transferComplete(self)
def clientConnectionFailed(self, conector, reason):
if isinstance(reason, ConnectionLost):
log.error('Connection failed: {}'.format(reason))
ReconnectingClientFactory.clientConnectionFailed(self, conector, reason)
def __str__(self):
return '<FileReceiver {}>'.format(self.path)
__repr__ = __str__
def execute_command(server_config, command, **kwargs):
before_fds = _get_open_fds() # see the comment in the finally clause below
if isinstance(command, basestring):
command = Command(command, **kwargs)
timeout = 300 if command.name in ['stop', 'pull'] else 10
factory = CommandExecutorFactory(server_config, command, timeout)
# reactors aren't designed to be re-startable. In order to be
# able to call execute_command multiple times, we need to froce
# re-installation of the reactor; hence this hackery.
# TODO: look into implementing restartable reactors. According to the
# Twisted FAQ, there is no good reason why there isn't one:
# http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#WhycanttheTwistedsreactorberestarted
from twisted.internet import default
del sys.modules['twisted.internet.reactor']
default.install()
global reactor # pylint: disable=W0603
reactor = sys.modules['twisted.internet.reactor']
try:
reactor.connectTCP(server_config.host, server_config.port, factory)
reactor.run()
return factory.result
finally:
# re-startable reactor hack part 2.
# twisted hijacks SIGINT and doesn't bother to un-hijack it when the reactor
# stops. So we have to do it for it *rolls eye*.
import signal
signal.signal(signal.SIGINT, signal.default_int_handler)
# OK, the reactor is also leaking file descriptors. Tracking down all
# of them is non trivial, so instead we're just comparing the before
# and after lists of open FDs for the current process, and closing all
# new ones, as execute_command should never leave anything open after
# it exits (even when downloading data files from the server).
# TODO: This is way too hacky even compared to the rest of this function.
# Additionally, the current implementation ties this to UNIX,
# so in the long run, we need to do this properly and get the FDs
# from the reactor.
after_fds = _get_open_fds()
for fd in (after_fds - before_fds):
try:
os.close(int(fd[1:]))
except OSError:
pass
# Below is the alternative code that gets FDs from the reactor, however
# at the moment it doesn't seem to get everything, which is why code
# above is used instead.
#for fd in readtor._selectables:
# os.close(fd)
#reactor._poller.close()
def _get_open_fds():
if os.name == 'posix':
import subprocess
pid = os.getpid()
procs = subprocess.check_output(
[ "lsof", '-w', '-Ff', "-p", str( pid ) ] )
return set(procs.split())
else:
# TODO: Implement the Windows equivalent.
return []
def run_send_command():
"""Main entry point when running as a script -- should not be invoked form another module."""
parser = get_config_parser()
parser.add_argument('command')
parser.add_argument('-o', '--output-directory', metavar='DIR', default='.',
help='Directory used to output data files (defaults to the current directory).')
parser.add_argument('--verbose', help='Produce verobose output.', action='store_true', default=False)
args = parser.parse_args()
if not args.device_config.labels:
args.device_config.labels = ['PORT_{}'.format(i) for i in xrange(len(args.device_config.resistor_values))]
if args.verbose:
log.start_logging('DEBUG')
else:
log.start_logging('INFO', fmt='%(levelname)-8s %(message)s')
if args.command == 'configure':
args.device_config.validate()
command = Command(args.command, config=args.device_config)
elif args.command == 'get_data':
command = Command(args.command, output_directory=args.output_directory)
else:
command = Command(args.command)
result = execute_command(args.server_config, command)
print result
if result.data:
print result.data
if __name__ == '__main__':
run_send_command()

View File

@@ -0,0 +1,99 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=E1101
import json
class Serializer(json.JSONEncoder):
def default(self, o): # pylint: disable=E0202
if isinstance(o, Serializable):
return o.serialize()
if isinstance(o, Enum.EnumEntry):
return o.name
return json.JSONEncoder.default(self, o)
class Serializable(object):
@classmethod
def deserialize(cls, text):
return cls(**json.loads(text))
def serialize(self, d=None):
if d is None:
d = self.__dict__
return json.dumps(d, cls=Serializer)
class DaqServerRequest(Serializable):
def __init__(self, command, params=None): # pylint: disable=W0231
self.command = command
self.params = params or {}
class DaqServerResponse(Serializable):
def __init__(self, status, message=None, data=None): # pylint: disable=W0231
self.status = status
self.message = message.strip().replace('\r\n', ' ') if message else ''
self.data = data or {}
def __str__(self):
return '{} {}'.format(self.status, self.message or '')
class Enum(object):
"""
Assuming MyEnum = Enum('A', 'B'),
MyEnum.A and MyEnum.B are valid values.
a = MyEnum.A
(a == MyEnum.A) == True
(a in MyEnum) == True
MyEnum('A') == MyEnum.A
str(MyEnum.A) == 'A'
"""
class EnumEntry(object):
def __init__(self, name):
self.name = name
def __str__(self):
return self.name
def __cmp__(self, other):
return cmp(self.name, str(other))
def __init__(self, *args):
for a in args:
setattr(self, a, self.EnumEntry(a))
def __call__(self, value):
if value not in self.__dict__:
raise ValueError('Not enum value: {}'.format(value))
return self.__dict__[value]
def __iter__(self):
for e in self.__dict__:
yield self.__dict__[e]
Status = Enum('OK', 'OKISH', 'ERROR')

View File

@@ -0,0 +1,154 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
from daqpower.common import Serializable
class ConfigurationError(Exception):
"""Raised when configuration passed into DaqServer is invaid."""
pass
class DeviceConfiguration(Serializable):
"""Encapulates configuration for the DAQ, typically, passed from
the client."""
valid_settings = ['device_id', 'v_range', 'dv_range', 'sampling_rate', 'resistor_values', 'labels']
default_device_id = 'Dev1'
default_v_range = 2.5
default_dv_range = 0.2
default_sampling_rate = 10000
# Channel map used in DAQ 6363 and similar.
default_channel_map = (0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23)
@property
def number_of_ports(self):
return len(self.resistor_values)
def __init__(self, **kwargs): # pylint: disable=W0231
try:
self.device_id = kwargs.pop('device_id') or self.default_device_id
self.v_range = float(kwargs.pop('v_range') or self.default_v_range)
self.dv_range = float(kwargs.pop('dv_range') or self.default_dv_range)
self.sampling_rate = int(kwargs.pop('sampling_rate') or self.default_sampling_rate)
self.resistor_values = kwargs.pop('resistor_values') or []
self.channel_map = kwargs.pop('channel_map') or self.default_channel_map
self.labels = (kwargs.pop('labels') or
['PORT_{}.csv'.format(i) for i in xrange(len(self.resistor_values))])
except KeyError, e:
raise ConfigurationError('Missing config: {}'.format(e.message))
if kwargs:
raise ConfigurationError('Unexpected config: {}'.format(kwargs))
def validate(self):
if not self.number_of_ports:
raise ConfigurationError('No resistor values were specified.')
if not len(self.resistor_values) == len(self.labels):
message = 'The number of resistors ({}) does not match the number of labels ({})'
raise ConfigurationError(message.format(len(self.resistor_values), len(self.labels)))
def __str__(self):
return self.serialize()
__repr__ = __str__
class ServerConfiguration(object):
"""Client-side server configuration."""
valid_settings = ['host', 'port']
default_host = '127.0.0.1'
default_port = 45677
def __init__(self, **kwargs):
self.host = kwargs.pop('host', None) or self.default_host
self.port = kwargs.pop('port', None) or self.default_port
if kwargs:
raise ConfigurationError('Unexpected config: {}'.format(kwargs))
def validate(self):
if not self.host:
raise ConfigurationError('Server host not specified.')
if not self.port:
raise ConfigurationError('Server port not specified.')
elif not isinstance(self.port, int):
raise ConfigurationError('Server port must be an integer.')
class UpdateDeviceConfig(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setting = option_string.strip('-').replace('-', '_')
if setting not in DeviceConfiguration.valid_settings:
raise ConfigurationError('Unkown option: {}'.format(option_string))
setattr(namespace._device_config, setting, values)
class UpdateServerConfig(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setting = option_string.strip('-').replace('-', '_')
if setting not in namespace.server_config.valid_settings:
raise ConfigurationError('Unkown option: {}'.format(option_string))
setattr(namespace.server_config, setting, values)
class ConfigNamespace(object):
class _N(object):
def __init__(self):
self.device_id = None
self.v_range = None
self.dv_range = None
self.sampling_rate = None
self.resistor_values = None
self.labels = None
self.channel_map = None
@property
def device_config(self):
return DeviceConfiguration(**self._device_config.__dict__)
def __init__(self):
self._device_config = self._N()
self.server_config = ServerConfiguration()
class ConfigArgumentParser(argparse.ArgumentParser):
def parse_args(self, *args, **kwargs):
kwargs['namespace'] = ConfigNamespace()
return super(ConfigArgumentParser, self).parse_args(*args, **kwargs)
def get_config_parser(server=True, device=True):
parser = ConfigArgumentParser()
if device:
parser.add_argument('--device-id', action=UpdateDeviceConfig)
parser.add_argument('--v-range', action=UpdateDeviceConfig, type=float)
parser.add_argument('--dv-range', action=UpdateDeviceConfig, type=float)
parser.add_argument('--sampling-rate', action=UpdateDeviceConfig, type=int)
parser.add_argument('--resistor-values', action=UpdateDeviceConfig, type=float, nargs='*')
parser.add_argument('--labels', action=UpdateDeviceConfig, nargs='*')
if server:
parser.add_argument('--host', action=UpdateServerConfig)
parser.add_argument('--port', action=UpdateServerConfig, type=int)
return parser

View File

@@ -0,0 +1,265 @@
# Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Creates a new DAQ device class. This class assumes that there is a
DAQ connected and mapped as Dev1. It assumes a specific syndesmology on the DAQ (it is not
meant to be a generic DAQ interface). The following diagram shows the wiring for one DaqDevice
port::
Port 0
========
| A0+ <--- Vr -------------------------|
| |
| A0- <--- GND -------------------// |
| |
| A1+ <--- V+ ------------|-------V+ |
| r | |
| A1- <--- Vr --/\/\/\----| |
| | |
| | |
| |--------------------------|
========
:number_of_ports: The number of ports connected on the DAQ. Each port requires 2 DAQ Channels
one for the source voltage and one for the Voltage drop over the
resistor r (V+ - Vr) allows us to detect the current.
:resistor_value: The resistance of r. Typically a few milliOhm
:downsample: The number of samples combined to create one Power point. If set to one
each sample corresponds to one reported power point.
:sampling_rate: The rate at which DAQ takes a sample from each channel.
"""
# pylint: disable=F0401,E1101,W0621
import os
import sys
import csv
import time
import threading
from Queue import Queue, Empty
import numpy
from PyDAQmx import Task
from PyDAQmx.DAQmxFunctions import DAQmxGetSysDevNames
from PyDAQmx.DAQmxTypes import int32, byref, create_string_buffer
from PyDAQmx.DAQmxConstants import (DAQmx_Val_Diff, DAQmx_Val_Volts, DAQmx_Val_GroupByScanNumber, DAQmx_Val_Auto,
DAQmx_Val_Acquired_Into_Buffer, DAQmx_Val_Rising, DAQmx_Val_ContSamps)
from daqpower import log
def list_available_devices():
"""Returns the list of DAQ devices visible to the driver."""
bufsize = 2048 # Should be plenty for all but the most pathalogical of situations.
buf = create_string_buffer('\000' * bufsize)
DAQmxGetSysDevNames(buf, bufsize)
return buf.value.split(',')
class ReadSamplesTask(Task):
def __init__(self, config, consumer):
Task.__init__(self)
self.config = config
self.consumer = consumer
self.sample_buffer_size = (self.config.sampling_rate + 1) * self.config.number_of_ports * 2
self.samples_read = int32()
self.remainder = []
# create voltage channels
for i in xrange(0, 2 * self.config.number_of_ports, 2):
self.CreateAIVoltageChan('{}/ai{}'.format(config.device_id, config.channel_map[i]),
'', DAQmx_Val_Diff,
-config.v_range, config.v_range,
DAQmx_Val_Volts, None)
self.CreateAIVoltageChan('{}/ai{}'.format(config.device_id, config.channel_map[i + 1]),
'', DAQmx_Val_Diff,
-config.dv_range, config.dv_range,
DAQmx_Val_Volts, None)
# configure sampling rate
self.CfgSampClkTiming('',
self.config.sampling_rate,
DAQmx_Val_Rising,
DAQmx_Val_ContSamps,
self.config.sampling_rate)
# register callbacks
self.AutoRegisterEveryNSamplesEvent(DAQmx_Val_Acquired_Into_Buffer, self.config.sampling_rate // 2, 0)
self.AutoRegisterDoneEvent(0)
def EveryNCallback(self):
samples_buffer = numpy.zeros((self.sample_buffer_size,), dtype=numpy.float64)
self.ReadAnalogF64(DAQmx_Val_Auto, 0.0, DAQmx_Val_GroupByScanNumber, samples_buffer,
self.sample_buffer_size, byref(self.samples_read), None)
self.consumer.write((samples_buffer, self.samples_read.value))
def DoneCallback(self, status): # pylint: disable=W0613,R0201
return 0 # The function should return an integer
class AsyncWriter(threading.Thread):
def __init__(self, wait_period=1):
super(AsyncWriter, self).__init__()
self.daemon = True
self.wait_period = wait_period
self.running = threading.Event()
self._stop_signal = threading.Event()
self._queue = Queue()
def write(self, stuff):
if self._stop_signal.is_set():
raise IOError('Attempting to writer to {} after it has been closed.'.format(self.__class__.__name__))
self._queue.put(stuff)
def do_write(self, stuff):
raise NotImplementedError()
def run(self):
self.running.set()
while True:
if self._stop_signal.is_set() and self._queue.empty():
break
try:
self.do_write(self._queue.get(block=True, timeout=self.wait_period))
except Empty:
pass # carry on
self.running.clear()
def stop(self):
self._stop_signal.set()
def wait(self):
while self.running.is_set():
time.sleep(self.wait_period)
class PortWriter(object):
def __init__(self, path):
self.path = path
self.fh = open(path, 'w', 0)
self.writer = csv.writer(self.fh)
self.writer.writerow(['power', 'voltage'])
def write(self, row):
self.writer.writerow(row)
def close(self):
self.fh.close()
def __del__(self):
self.close()
class SamplePorcessorError(Exception):
pass
class SampleProcessor(AsyncWriter):
def __init__(self, resistor_values, output_directory, labels):
super(SampleProcessor, self).__init__()
self.resistor_values = resistor_values
self.output_directory = output_directory
self.labels = labels
self.number_of_ports = len(resistor_values)
if len(self.labels) != self.number_of_ports:
message = 'Number of labels ({}) does not match number of ports ({}).'
raise SamplePorcessorError(message.format(len(self.labels), self.number_of_ports))
self.port_writers = []
def do_write(self, sample_tuple):
samples, number_of_samples = sample_tuple
for i in xrange(0, number_of_samples * self.number_of_ports * 2, self.number_of_ports * 2):
for j in xrange(self.number_of_ports):
V = float(samples[i + 2 * j])
DV = float(samples[i + 2 * j + 1])
P = V * (DV / self.resistor_values[j])
self.port_writers[j].write([P, V])
def start(self):
for label in self.labels:
port_file = self.get_port_file_path(label)
writer = PortWriter(port_file)
self.port_writers.append(writer)
super(SampleProcessor, self).start()
def stop(self):
super(SampleProcessor, self).stop()
self.wait()
for writer in self.port_writers:
writer.close()
def get_port_file_path(self, port_id):
if port_id in self.labels:
return os.path.join(self.output_directory, port_id + '.csv')
else:
raise SamplePorcessorError('Invalid port ID: {}'.format(port_id))
def __del__(self):
self.stop()
class DaqRunner(object):
@property
def number_of_ports(self):
return self.config.number_of_ports
def __init__(self, config, output_directory):
self.config = config
self.processor = SampleProcessor(config.resistor_values, output_directory, config.labels)
self.task = ReadSamplesTask(config, self.processor)
self.is_running = False
def start(self):
log.debug('Starting sample processor.')
self.processor.start()
log.debug('Starting DAQ Task.')
self.task.StartTask()
self.is_running = True
log.debug('Runner started.')
def stop(self):
self.is_running = False
log.debug('Stopping DAQ Task.')
self.task.StopTask()
log.debug('Stopping sample processor.')
self.processor.stop()
log.debug('Runner stopped.')
def get_port_file_path(self, port_id):
return self.processor.get_port_file_path(port_id)
if __name__ == '__main__':
from collections import namedtuple
DeviceConfig = namedtuple('DeviceConfig', ['device_id', 'channel_map', 'resistor_values',
'v_range', 'dv_range', 'sampling_rate',
'number_of_ports', 'labels'])
channel_map = (0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23)
resistor_values = [0.005]
labels = ['PORT_0']
dev_config = DeviceConfig('Dev1', channel_map, resistor_values, 2.5, 0.2, 10000, len(resistor_values), labels)
if not len(sys.argv) == 3:
print 'Usage: {} OUTDIR DURATION'.format(os.path.basename(__file__))
sys.exit(1)
output_directory = sys.argv[1]
duration = float(sys.argv[2])
print "Avialable devices:", list_availabe_devices()
runner = DaqRunner(dev_config, output_directory)
runner.start()
time.sleep(duration)
runner.stop()

View File

@@ -0,0 +1,53 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
from twisted.python import log
__all__ = ['debug', 'info', 'warning', 'error', 'critical', 'start_logging']
debug = lambda x: log.msg(x, logLevel=logging.DEBUG)
info = lambda x: log.msg(x, logLevel=logging.INFO)
warning = lambda x: log.msg(x, logLevel=logging.WARNING)
error = lambda x: log.msg(x, logLevel=logging.ERROR)
critical = lambda x: log.msg(x, logLevel=logging.CRITICAL)
class CustomLoggingObserver(log.PythonLoggingObserver):
def emit(self, eventDict):
if 'logLevel' in eventDict:
level = eventDict['logLevel']
elif eventDict['isError']:
level = logging.ERROR
else:
# All of that just just to override this one line from
# default INFO level...
level = logging.DEBUG
text = log.textFromEventDict(eventDict)
if text is None:
return
self.logger.log(level, text)
logObserver = CustomLoggingObserver()
logObserver.start()
def start_logging(level, fmt='%(asctime)s %(levelname)-8s: %(message)s'):
logging.basicConfig(level=getattr(logging, level), format=fmt)

View File

@@ -0,0 +1,480 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=E1101,W0613
from __future__ import division
import os
import sys
import socket
import argparse
import shutil
import time
from datetime import datetime
from zope.interface import implements
from twisted.protocols.basic import LineReceiver
from twisted.internet.protocol import Factory, Protocol
from twisted.internet import reactor, interfaces
from twisted.internet.error import ConnectionLost, ConnectionDone
if __name__ == "__main__": # for debugging
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from daqpower import log
from daqpower.config import DeviceConfiguration
from daqpower.common import DaqServerRequest, DaqServerResponse, Status
try:
from daqpower.daq import DaqRunner, list_available_devices
except ImportError:
# May be using debug mode.
DaqRunner = None
list_available_devices = lambda : ['Dev1']
class ProtocolError(Exception):
pass
class DummyDaqRunner(object):
"""Dummy stub used when running in debug mode."""
num_rows = 200
@property
def number_of_ports(self):
return self.config.number_of_ports
def __init__(self, config, output_directory):
log.info('Creating runner with {} {}'.format(config, output_directory))
self.config = config
self.output_directory = output_directory
self.is_running = False
def start(self):
import csv, random
log.info('runner started')
for i in xrange(self.config.number_of_ports):
rows = [['power', 'voltage']] + [[random.gauss(1.0, 1.0), random.gauss(1.0, 0.1)]
for j in xrange(self.num_rows)]
with open(self.get_port_file_path(self.config.labels[i]), 'wb') as wfh:
writer = csv.writer(wfh)
writer.writerows(rows)
self.is_running = True
def stop(self):
self.is_running = False
log.info('runner stopped')
def get_port_file_path(self, port_id):
if port_id in self.config.labels:
return os.path.join(self.output_directory, '{}.csv'.format(port_id))
else:
raise Exception('Invalid port id: {}'.format(port_id))
class DaqServer(object):
def __init__(self, base_output_directory):
self.base_output_directory = os.path.abspath(base_output_directory)
if os.path.isdir(self.base_output_directory):
log.info('Using output directory: {}'.format(self.base_output_directory))
else:
log.info('Creating new output directory: {}'.format(self.base_output_directory))
os.makedirs(self.base_output_directory)
self.runner = None
self.output_directory = None
self.labels = None
def configure(self, config_string):
message = None
if self.runner:
message = 'Configuring a new session before previous session has been terminated.'
log.warning(message)
if self.runner.is_running:
self.runner.stop()
config = DeviceConfiguration.deserialize(config_string)
config.validate()
self.output_directory = self._create_output_directory()
self.labels = config.labels
log.info('Writing port files to {}'.format(self.output_directory))
self.runner = DaqRunner(config, self.output_directory)
return message
def start(self):
if self.runner:
if not self.runner.is_running:
self.runner.start()
else:
message = 'Calling start() before stop() has been called. Data up to this point will be lost.'
log.warning(message)
self.runner.stop()
self.runner.start()
return message
else:
raise ProtocolError('Start called before a session has been configured.')
def stop(self):
if self.runner:
if self.runner.is_running:
self.runner.stop()
else:
message = 'Attempting to stop() before start() was invoked.'
log.warning(message)
self.runner.stop()
return message
else:
raise ProtocolError('Stop called before a session has been configured.')
def list_devices(self):
return list_available_devices()
def list_ports(self):
return self.labels
def list_port_files(self):
if not self.runner:
raise ProtocolError('Attempting to list port files before session has been configured.')
ports_with_files = []
for port_id in self.labels:
path = self.get_port_file_path(port_id)
if os.path.isfile(path):
ports_with_files.append(port_id)
return ports_with_files
def get_port_file_path(self, port_id):
if not self.runner:
raise ProtocolError('Attepting to get port file path before session has been configured.')
return self.runner.get_port_file_path(port_id)
def terminate(self):
message = None
if self.runner:
if self.runner.is_running:
message = 'Terminating session before runner has been stopped.'
log.warning(message)
self.runner.stop()
self.runner = None
if self.output_directory and os.path.isdir(self.output_directory):
shutil.rmtree(self.output_directory)
self.output_directory = None
log.info('Session terminated.')
else: # Runner has not been created.
message = 'Attempting to close session before it has been configured.'
log.warning(message)
return message
def _create_output_directory(self):
basename = datetime.now().strftime('%Y-%m-%d_%H%M%S%f')
dirname = os.path.join(self.base_output_directory, basename)
os.makedirs(dirname)
return dirname
def __del__(self):
if self.runner:
self.runner.stop()
def __str__(self):
return '({})'.format(self.base_output_directory)
__repr__ = __str__
class DaqControlProtocol(LineReceiver): # pylint: disable=W0223
def __init__(self, daq_server):
self.daq_server = daq_server
self.factory = None
def lineReceived(self, line):
line = line.strip()
log.info('Received: {}'.format(line))
try:
request = DaqServerRequest.deserialize(line)
except Exception, e: # pylint: disable=W0703
self.sendError('Received bad request ({}: {})'.format(e.__class__.__name__, e.message))
else:
self.processRequest(request)
def processRequest(self, request):
try:
if request.command == 'configure':
self.configure(request)
elif request.command == 'start':
self.start(request)
elif request.command == 'stop':
self.stop(request)
elif request.command == 'list_devices':
self.list_devices(request)
elif request.command == 'list_ports':
self.list_ports(request)
elif request.command == 'list_port_files':
self.list_port_files(request)
elif request.command == 'pull':
self.pull_port_data(request)
elif request.command == 'close':
self.terminate(request)
else:
self.sendError('Received unknown command: {}'.format(request.command))
except Exception, e: # pylint: disable=W0703
self.sendError('{}: {}'.format(e.__class__.__name__, e.message))
def configure(self, request):
if 'config' in request.params:
result = self.daq_server.configure(request.params['config'])
if not result:
self.sendResponse(Status.OK)
else:
self.sendResponse(Status.OKISH, message=result)
else:
self.sendError('Invalid config; config string not provided.')
def start(self, request):
result = self.daq_server.start()
if not result:
self.sendResponse(Status.OK)
else:
self.sendResponse(Status.OKISH, message=result)
def stop(self, request):
result = self.daq_server.stop()
if not result:
self.sendResponse(Status.OK)
else:
self.sendResponse(Status.OKISH, message=result)
def pull_port_data(self, request):
if 'port_id' in request.params:
port_id = request.params['port_id']
port_file = self.daq_server.get_port_file_path(port_id)
if os.path.isfile(port_file):
port = self._initiate_file_transfer(port_file)
self.sendResponse(Status.OK, data={'port_number': port})
else:
self.sendError('File for port {} does not exist.'.format(port_id))
else:
self.sendError('Invalid pull request; port id not provided.')
def list_devices(self, request):
devices = self.daq_server.list_devices()
self.sendResponse(Status.OK, data={'devices': devices})
def list_ports(self, request):
port_labels = self.daq_server.list_ports()
self.sendResponse(Status.OK, data={'ports': port_labels})
def list_port_files(self, request):
port_labels = self.daq_server.list_port_files()
self.sendResponse(Status.OK, data={'ports': port_labels})
def terminate(self, request):
status = Status.OK
message = ''
if self.factory.transfer_sessions:
message = 'Terminating with file tranfer sessions in progress. '
log.warning(message)
for session in self.factory.transfer_sessions:
self.factory.transferComplete(session)
message += self.daq_server.terminate() or ''
if message:
status = Status.OKISH
self.sendResponse(status, message)
def sendError(self, message):
log.error(message)
self.sendResponse(Status.ERROR, message)
def sendResponse(self, status, message=None, data=None):
response = DaqServerResponse(status, message=message, data=data)
self.sendLine(response.serialize())
def sendLine(self, line):
log.info('Responding: {}'.format(line))
LineReceiver.sendLine(self, line.replace('\r\n',''))
def _initiate_file_transfer(self, filepath):
sender_factory = FileSenderFactory(filepath, self.factory)
connector = reactor.listenTCP(0, sender_factory)
self.factory.transferInitiated(sender_factory, connector)
return connector.getHost().port
class DaqFactory(Factory):
protocol = DaqControlProtocol
check_alive_period = 5 * 60
max_transfer_lifetime = 30 * 60
def __init__(self, server):
self.server = server
self.transfer_sessions = {}
def buildProtocol(self, addr):
proto = DaqControlProtocol(self.server)
proto.factory = self
reactor.callLater(self.check_alive_period, self.pulse)
return proto
def clientConnectionLost(self, connector, reason):
log.msg('client connection lost: {}.'.format(reason))
if not isinstance(reason, ConnectionLost):
log.msg('ERROR: Client terminated connection mid-transfer.')
for session in self.transfer_sessions:
self.transferComplete(session)
def transferInitiated(self, session, connector):
self.transfer_sessions[session] = (time.time(), connector)
def transferComplete(self, session, reason='OK'):
if reason != 'OK':
log.error(reason)
self.transfer_sessions[session][1].stopListening()
del self.transfer_sessions[session]
def pulse(self):
"""Close down any file tranfer sessions that have been open for too long."""
current_time = time.time()
for session in self.transfer_sessions:
start_time, conn = self.transfer_sessions[session]
if (current_time - start_time) > self.max_transfer_lifetime:
message = '{} session on port {} timed out'
self.transferComplete(session, message.format(session, conn.getHost().port))
if self.transfer_sessions:
reactor.callLater(self.check_alive_period, self.pulse)
def __str__(self):
return '<DAQ {}>'.format(self.server)
__repr__ = __str__
class FileReader(object):
implements(interfaces.IPushProducer)
def __init__(self, filepath):
self.fh = open(filepath)
self.proto = None
self.done = False
self._paused = True
def setProtocol(self, proto):
self.proto = proto
def resumeProducing(self):
if not self.proto:
raise ProtocolError('resumeProducing called with no protocol set.')
self._paused = False
try:
while not self._paused:
line = self.fh.next().rstrip('\n') + '\r\n'
self.proto.transport.write(line)
except StopIteration:
log.debug('Sent everything.')
self.stopProducing()
def pauseProducing(self):
self._paused = True
def stopProducing(self):
self.done = True
self.fh.close()
self.proto.transport.unregisterProducer()
self.proto.transport.loseConnection()
class FileSenderProtocol(Protocol):
def __init__(self, reader):
self.reader = reader
self.factory = None
def connectionMade(self):
self.transport.registerProducer(self.reader, True)
self.reader.resumeProducing()
def connectionLost(self, reason=ConnectionDone):
if self.reader.done:
self.factory.transferComplete()
else:
self.reader.pauseProducing()
self.transport.unregisterProducer()
class FileSenderFactory(Factory):
@property
def done(self):
if self.reader:
return self.reader.done
else:
return None
def __init__(self, path, owner):
self.path = os.path.abspath(path)
self.reader = None
self.owner = owner
def buildProtocol(self, addr):
if not self.reader:
self.reader = FileReader(self.path)
proto = FileSenderProtocol(self.reader)
proto.factory = self
self.reader.setProtocol(proto)
return proto
def transferComplete(self):
self.owner.transferComplete(self)
def __hash__(self):
return hash(self.path)
def __str__(self):
return '<FileSender {}>'.format(self.path)
__repr__ = __str__
def run_server():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--directory', help='Working directory', metavar='DIR', default='.')
parser.add_argument('-p', '--port', help='port the server will listen on.',
metavar='PORT', default=45677, type=int)
parser.add_argument('--debug', help='Run in debug mode (no DAQ connected).',
action='store_true', default=False)
parser.add_argument('--verbose', help='Produce verobose output.', action='store_true', default=False)
args = parser.parse_args()
if args.debug:
global DaqRunner # pylint: disable=W0603
DaqRunner = DummyDaqRunner
else:
if not DaqRunner:
raise ImportError('DaqRunner')
if args.verbose or args.debug:
log.start_logging('DEBUG')
else:
log.start_logging('INFO')
server = DaqServer(args.directory)
reactor.listenTCP(args.port, DaqFactory(server)).getHost()
hostname = socket.gethostbyname(socket.gethostname())
log.info('Listening on {}:{}'.format(hostname, args.port))
reactor.run()
if __name__ == "__main__":
run_server()

View File

@@ -0,0 +1,3 @@
#!/usr/bin/env python
from daqpower.server import run_server
run_server()

View File

@@ -0,0 +1,3 @@
#!/usr/bin/env python
from daqpower.client import run_send_command
run_send_command()

52
wlauto/external/daq_server/src/setup.py vendored Normal file
View File

@@ -0,0 +1,52 @@
# Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import warnings
from distutils.core import setup
import daqpower
warnings.filterwarnings('ignore', "Unknown distribution option: 'install_requires'")
params = dict(
name='daqpower',
version=daqpower.__version__,
packages=[
'daqpower',
],
scripts=[
'scripts/run-daq-server',
'scripts/send-daq-command',
],
url='N/A',
maintainer='workload-automation',
maintainer_email='workload-automation@arm.com',
install_requires=[
'twisted',
'PyDAQmx',
],
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[
'Development Status :: 3 - Alpha',
'Environment :: Console',
'License :: Other/Proprietary License',
'Operating System :: Unix',
'Programming Language :: Python :: 2.7',
],
)
setup(**params)