diff --git a/wa/framework/getters.py b/wa/framework/getters.py index a34bfa19..9ce5f94e 100644 --- a/wa/framework/getters.py +++ b/wa/framework/getters.py @@ -71,6 +71,22 @@ def get_generic_resource(resource, files): return matches[0] +def get_from_location(basepath, resource): + if resource.kind == 'file': + path = os.path.join(basepath, resource.path) + if os.path.exists(path): + return path + elif resource.kind == 'executable': + path = os.path.join(basepath, 'bin', resource.abi, resource.filename) + if os.path.exists(path): + return path + elif resource.kind in ['apk', 'jar', 'revent']: + files = get_by_extension(basepath, resource.kind) + return get_generic_resource(resource, files) + + return None + + class Package(ResourceGetter): name = 'package' @@ -84,19 +100,232 @@ class Package(ResourceGetter): else: modname = resource.owner.__module__ basepath = os.path.dirname(sys.modules[modname].__file__) + return get_from_location(basepath, resource) - if resource.kind == 'file': - path = os.path.join(basepath, resource.path) - if os.path.exists(path): - return path + +class UserDirectory(ResourceGetter): + + name = 'user' + + def register(self, resolver): + resolver.register(self.get, SourcePriority.local) + + def get(self, resource): + basepath = settings.dependencies_directory + return get_from_location(basepath, resource) + + +class Http(ResourceGetter): + + name = 'http' + description = """ + Downloads resources from a server based on an index fetched from the + specified URL. + + Given a URL, this will try to fetch ``/index.json``. The index file + maps extension names to a list of corresponing asset descriptons. Each + asset description continas a path (relative to the base URL) of the + resource and a SHA256 hash, so that this Getter can verify whether the + resource on the remote has changed. + + For example, let's assume we want to get the APK file for workload "foo", + and that assets are hosted at ``http://example.com/assets``. This Getter + will first try to donwload ``http://example.com/assests/index.json``. The + index file may contian something like :: + + { + "foo": [ + { + "path": "foo-app.apk", + "sha256": "b14530bb47e04ed655ac5e80e69beaa61c2020450e18638f54384332dffebe86" + }, + { + "path": "subdir/some-other-asset.file", + "sha256": "48d9050e9802246d820625717b72f1c2ba431904b8484ca39befd68d1dbedfff" + } + ] + } + + This Getter will look through the list of assets for "foo" (in this case, + two) check the paths until it finds one matching the resource (in this + case, "foo-app.apk"). Finally, it will try to dowload that file relative + to the base URL and extension name (in this case, + "http://example.com/assets/foo/foo-app.apk"). The downloaded version will + be cached locally, so that in the future, the getter will check the SHA256 + hash of the local file against the one advertised inside index.json, and + provided that hasn't changed, it won't try to download the file again. + + """ + parameters = [ + Parameter('url', global_alias='remote_assets_url', + description=""" + URL of the index file for assets on an HTTP server. + """), + Parameter('username', + description=""" + User name for authenticating with assets URL + """), + Parameter('password', + description=""" + Password for authenticationg with assets URL + """), + Parameter('always_fetch', kind=boolean, default=False, + global_alias='always_fetch_remote_assets', + description=""" + If ``True``, will always attempt to fetch assets from the + remote, even if a local cached copy is available. + """), + Parameter('chunk_size', kind=int, default=1024, + description=""" + Chunk size for streaming large assets. + """), + ] + + def __init__(self, **kwargs): + super(Http, self).__init__(**kwargs) + self.logger = logger + self.index = None + + def register(self, resolver): + resolver.register(self.get, SourcePriority.remote) + + def get(self, resource): + if not resource.owner: + return # TODO: add support for unowned resources + if not self.index: + self.index = self.fetch_index() + asset = self.resolve_resource(resource) + if not asset: + return + return self.download_asset(asset, resource.owner.name) + + def fetch_index(self): + if not self.url: + return {} + index_url = urljoin(self.url, 'index.json') + response = self.geturl(index_url) + if response.status_code != httplib.OK: + message = 'Could not fetch "{}"; recieved "{} {}"' + self.logger.error(message.format(index_url, + response.status_code, + response.reason)) + return {} + return json.loads(response.content) + + def download_asset(self, asset, owner_name): + url = urljoin(self.url, owner_name, asset['path']) + local_path = _f(os.path.join(settings.dependencies_directory, '__remote', + owner_name, asset['path'].replace('/', os.sep))) + if os.path.exists(local_path) and not self.always_fetch: + local_sha = sha256(local_path) + if local_sha == asset['sha256']: + self.logger.debug('Local SHA256 matches; not re-downloading') + return local_path + self.logger.debug('Downloading {}'.format(url)) + response = self.geturl(url, stream=True) + if response.status_code != httplib.OK: + message = 'Could not download asset "{}"; recieved "{} {}"' + self.logger.warning(message.format(url, + response.status_code, + response.reason)) + return + with open(local_path, 'wb') as wfh: + for chunk in response.iter_content(chunk_size=self.chunk_size): + wfh.write(chunk) + return local_path + + def geturl(self, url, stream=False): + if self.username: + auth = (self.username, self.password) + else: + auth = None + return requests.get(url, auth=auth, stream=stream) + + def resolve_resource(self, resource): + # pylint: disable=too-many-branches,too-many-locals + assets = self.index.get(resource.owner.name, {}) + if not assets: + return {} + + asset_map = {a['path']: a for a in assets} + if resource.kind in ['apk', 'jar', 'revent']: + if resource.kind == 'apk' and resource.version: + # TODO: modify the index format to attach version info to the + # APK entries. + msg = 'Versions of APKs cannot be fetched over HTTP at this time' + self.logger.warning(msg) + return {} + path = get_generic_resource(resource, asset_map.keys()) + if path: + return asset_map[path] elif resource.kind == 'executable': - path = os.path.join(basepath, 'bin', resource.abi, resource.filename) - if os.path.exists(path): - return path - elif resource.kind in ['apk', 'jar', 'revent']: - files = get_by_extension(basepath, resource.kind) - return get_generic_resource(resource, files) - - return None + path = '/'.join(['bin', resource.abi, resource.filename]) + for asset in assets: + if asset['path'].lower() == path.lower(): + return asset + else: # file + for asset in assets: + if asset['path'].lower() == resource.path.lower(): + return asset +class Filer(ResourceGetter): + + name = 'filer' + description = """ + Finds resources on a (locally mounted) remote filer and caches them + locally. + + This assumes that the filer is mounted on the local machine (e.g. as a + samba share). + + """ + parameters = [ + Parameter('remote_path', global_alias='remote_assets_path', default='', + description=""" + Path, on the local system, where the assets are located. + """), + Parameter('always_fetch', kind=boolean, default=False, + global_alias='always_fetch_remote_assets', + description=""" + If ``True``, will always attempt to fetch assets from the + remote, even if a local cached copy is available. + """), + ] + + def register(self, resolver): + resolver.register(self.get, SourcePriority.lan) + + def get(self, resource): + if resource.owner: + remote_path = os.path.join(self.remote_path, resource.owner.name) + local_path = os.path.join(settings.dependencies_directory, '__filer', + resource.owner.dependencies_directory) + return self.try_get_resource(resource, remote_path, local_path) + else: # No owner + result = None + for entry in os.listdir(remote_path): + remote_path = os.path.join(self.remote_path, entry) + local_path = os.path.join(settings.dependencies_directory, '__filer', + settings.dependencies_directory, entry) + result = self.try_get_resource(resource, remote_path, local_path) + if result: + break + return result + + def try_get_resource(self, resource, remote_path, local_path): + if not self.always_fetch: + result = get_from_location(local_path, resource) + if result: + return result + if remote_path: + # Didn't find it cached locally; now check the remoted + result = get_from_location(remote_path, resource) + if not result: + return result + else: # remote path is not set + return None + # Found it remotely, cache locally, then return it + local_full_path = os.path.join(_d(local_path), os.path.basename(result)) + self.logger.debug('cp {} {}'.format(result, local_full_path)) + shutil.copy(result, local_full_path) diff --git a/wa/utils/misc.py b/wa/utils/misc.py index bf6a28bb..bb81a0d6 100644 --- a/wa/utils/misc.py +++ b/wa/utils/misc.py @@ -588,10 +588,11 @@ def touch(path): def get_object_name(obj): if hasattr(obj, 'name'): return obj.name - elif hasattr(obj,'func_name'): - return obj.func_name elif hasattr(obj, 'im_func'): - return '{}.{}'.format(obj.im_class.__name__, obj.im_func.func_name) + return '{}.{}'.format(get_object_name(obj.im_class), + obj.im_func.func_name) + elif hasattr(obj, 'func_name'): + return obj.func_name elif hasattr(obj, '__name__'): return obj.__name__ elif hasattr(obj, '__class__'):