Source code for hubblestack.extmods.modules.hubble

# -*- encoding: utf-8 -*-
'''
Loader and primary interface for nova modules

See README for documentation

Configuration:
    - hubblestack:nova:module_dir
    - hubblestack:nova:profile_dir
    - hubblestack:nova:saltenv
    - hubblestack:nova:autoload
    - hubblestack:nova:autosync
'''
from __future__ import absolute_import
import logging

log = logging.getLogger(__name__)

import imp
import os
import sys
import six
import inspect
import yaml
import traceback

import salt
import salt.utils
from salt.exceptions import CommandExecutionError
from hubblestack import __version__
from hubblestack.extmods.modules.nova_loader import NovaLazyLoader

__nova__ = {}


[docs]def audit(configs=None, tags='*', verbose=None, show_success=None, show_compliance=None, show_profile=None, called_from_top=None, debug=None, labels=None, **kwargs): ''' Primary entry point for audit calls. configs List (comma-separated or python list) of yaml configs/directories to search for audit data. Directories are dot-separated, much in the same way as Salt states. For individual config names, leave the .yaml extension off. If a given path resolves to a python file, it will be treated as a single config. Otherwise it will be treated as a directory. All configs found in a recursive search of the specified directories will be included in the audit. If configs is not provided, this function will call ``hubble.top`` instead. tags Glob pattern string for tags to include in the audit. This way you can give a directory, and tell the system to only run the `CIS*`-tagged audits, for example. verbose Whether to show additional information about audits, including description, remediation instructions, etc. The data returned depends on the audit module. Defaults to False. Configurable via `hubblestack:nova:verbose` in minion config/pillar. show_success Whether to show successful audits in addition to failed audits. Defaults to True. Configurable via `hubblestack:nova:show_success` in minion config/pillar. show_compliance Whether to show compliance as a percentage (successful checks divided by total checks). Defaults to True. Configurable via `hubblestack:nova:show_compliance` in minion config/pillar. show_profile DEPRECATED called_from_top Ignore this argument. It is used for distinguishing between user-calls of this function and calls from hubble.top. debug Whether to log additional information to help debug nova. Defaults to False. Configurable via `hubblestack:nova:debug` in minion config/pillar. labels Tests with matching labels are executed. If multiple labels are passed, then tests which have all those labels are executed. **kwargs Any parameters & values that are not explicitly defined will be passed directly through to the Nova module(s). CLI Examples:: salt '*' hubble.audit foo salt '*' hubble.audit foo,bar tags='CIS*' salt '*' hubble.audit foo,bar.baz verbose=True ''' if configs is None: return top(verbose=verbose, show_success=show_success, show_compliance=show_compliance, labels=labels) if labels: if not isinstance(labels, list): labels=labels.split(',') if not called_from_top and __salt__['config.get']('hubblestack:nova:autoload', True): load() if not __nova__: return False, 'No nova modules/data have been loaded.' if verbose is None: verbose = __salt__['config.get']('hubblestack:nova:verbose', False) if show_success is None: show_success = __salt__['config.get']('hubblestack:nova:show_success', True) if show_compliance is None: show_compliance = __salt__['config.get']('hubblestack:nova:show_compliance', True) if show_profile is not None: log.warning( 'Keyword argument \'show_profile\' is no longer supported' ) if debug is None: debug = __salt__['config.get']('hubblestack:nova:debug', False) if not isinstance(configs, list): # Convert string configs = configs.split(',') # Convert config list to paths, with leading slashes configs = [os.path.join(os.path.sep, os.path.join(*(con.split('.yaml')[0]).split('.'))) for con in configs] # Pass any module parameters through to the Nova module nova_kwargs = {} # Get values from config first (if any) and merge into nova_kwargs nova_kwargs_config = __salt__['config.get']('hubblestack:nova:nova_kwargs', False) if nova_kwargs_config is not False: nova_kwargs.update(nova_kwargs_config) # Now process arguments from CLI and merge into nova_kwargs_dict if kwargs is not None: nova_kwargs.update(kwargs) log.debug('nova_kwargs: ' + str(nova_kwargs)) ret = _run_audit(configs, tags, debug, labels, **nova_kwargs) terse_results = {} verbose_results = {} # Pull out just the tag and description terse_results['Failure'] = [] tags_descriptions = set() for tag_data in ret.get('Failure', []): tag = tag_data['tag'] description = tag_data.get('description') if (tag, description) not in tags_descriptions: terse_results['Failure'].append({tag: description}) tags_descriptions.add((tag, description)) terse_results['Success'] = [] tags_descriptions = set() for tag_data in ret.get('Success', []): tag = tag_data['tag'] description = tag_data.get('description') if (tag, description) not in tags_descriptions: terse_results['Success'].append({tag: description}) tags_descriptions.add((tag, description)) terse_results['Controlled'] = [] control_reasons = set() for tag_data in ret.get('Controlled', []): tag = tag_data['tag'] control_reason = tag_data.get('control', '') description = tag_data.get('description') if (tag, description, control_reason) not in control_reasons: terse_results['Controlled'].append({tag: control_reason}) control_reasons.add((tag, description, control_reason)) # Calculate compliance level if show_compliance: compliance = _calculate_compliance(terse_results) else: compliance = False if not show_success and 'Success' in terse_results: terse_results.pop('Success') if not terse_results['Controlled']: terse_results.pop('Controlled') # Format verbose output as single-key dictionaries with tag as key if verbose: verbose_results['Failure'] = [] for tag_data in ret.get('Failure', []): tag = tag_data['tag'] verbose_results['Failure'].append({tag: tag_data}) verbose_results['Success'] = [] for tag_data in ret.get('Success', []): tag = tag_data['tag'] verbose_results['Success'].append({tag: tag_data}) if not show_success and 'Success' in verbose_results: verbose_results.pop('Success') verbose_results['Controlled'] = [] for tag_data in ret.get('Controlled', []): tag = tag_data['tag'] verbose_results['Controlled'].append({tag: tag_data}) if not verbose_results['Controlled']: verbose_results.pop('Controlled') results = verbose_results else: results = terse_results if compliance: results['Compliance'] = compliance if not called_from_top and not results: results['Messages'] = 'No audits matched this host in the specified profiles.' for error in ret.get('Errors', []): if 'Errors' not in results: results['Errors'] = [] results['Errors'].append(error) return results
def _run_audit(configs, tags, debug, labels, **kwargs): results = {} # Compile a list of audit data sets which we need to run to_run = set() for config in configs: found_for_config = False for key in __nova__.__data__: key_path_split = key.split('.yaml')[0].split(os.path.sep) matches = True if config != os.path.sep: for i, path in enumerate(config.split(os.path.sep)): if i >= len(key_path_split) or path != key_path_split[i]: matches = False if matches: # Found a match, add the audit data to the set found_for_config = True to_run.add(key) if not found_for_config: # No matches were found for this entry, add an error if 'Errors' not in results: results['Errors'] = [] results['Errors'].append({config: {'error': 'No matching profiles found for {0}' .format(config)}}) # compile list of tuples with profile name and profile data data_list = [(key.split('.yaml')[0].split(os.path.sep)[-1], __nova__.__data__[key]) for key in to_run] if debug: log.debug('hubble.py configs:') log.debug(configs) log.debug('hubble.py data_list:') log.debug(data_list) # Run the audits # This is currently pretty brute-force -- we just run all the modules we # have available with the data list, so data will be processed multiple # times. However, for the scale we're working at this should be fine. # We can revisit if this ever becomes a big bottleneck for key, func in __nova__._dict.iteritems(): try: ret = func(data_list, tags, labels, **kwargs) except Exception as exc: log.error('Exception occurred in nova module:') log.error(traceback.format_exc()) if 'Errors' not in results: results['Errors'] = [] results['Errors'].append({key: {'error': 'exception occurred', 'data': traceback.format_exc().splitlines()[-1]}}) continue else: if not isinstance(ret, dict): if 'Errors' not in results: results['Errors'] = [] results['Errors'].append({key: {'error': 'bad return type', 'data': ret}}) continue # Merge in the results for key, val in ret.iteritems(): if key not in results: results[key] = [] results[key].extend(val) processed_controls = {} # Inspect the data for compensating control data for _, audit_data in data_list: control_config = audit_data.get('control', []) for control in control_config: if isinstance(control, str): processed_controls[control] = {} else: # dict for control_tag, control_data in control.iteritems(): if isinstance(control_data, str): processed_controls[control_tag] = {'reason': control_data} else: # dict processed_controls[control_tag] = control_data if debug: log.debug('hubble.py control data:') log.debug(processed_controls) # Look through the failed results to find audits which match our control config failures_to_remove = [] for i, failure in enumerate(results.get('Failure', [])): failure_tag = failure['tag'] if failure_tag in processed_controls: failures_to_remove.append(i) if 'Controlled' not in results: results['Controlled'] = [] failure.update({ 'control': processed_controls[failure_tag].get('reason') }) results['Controlled'].append(failure) # Remove controlled failures from results['Failure'] if failures_to_remove: for failure_index in reversed(sorted(set(failures_to_remove))): results['Failure'].pop(failure_index) for key in results.keys(): if not results[key]: results.pop(key) return results
[docs]def top(topfile='top.nova', verbose=None, show_success=None, show_compliance=None, show_profile=None, debug=None, labels=None): ''' Compile and run all yaml data from the specified nova topfile. Nova topfiles look very similar to saltstack topfiles, except the top-level key is always ``nova``, as nova doesn't have a concept of environments. .. code-block:: yaml nova: '*': - cve_scan - cis_gen 'web*': - firewall - cis-centos-7-l2-scored - cis-centos-7-apache24-l1-scored 'G@os_family:debian': - netstat - cis-debian-7-l2-scored: 'CIS*' - cis-debian-7-mysql57-l1-scored: 'CIS 2.1.2' Additionally, all nova topfile matches are compound matches, so you never need to define a match type like you do in saltstack topfiles. Each list item is a string representing the dot-separated location of a yaml file which will be run with hubble.audit. You can also specify a tag glob to use as a filter for just that yaml file, using a colon after the yaml file (turning it into a dictionary). See the last two lines in the yaml above for examples. Arguments: topfile The path of the topfile, relative to your hubblestack_nova_profiles directory. verbose Whether to show additional information about audits, including description, remediation instructions, etc. The data returned depends on the audit module. Defaults to False. Configurable via `hubblestack:nova:verbose` in minion config/pillar. show_success Whether to show successful audits in addition to failed audits. Defaults to True. Configurable via `hubblestack:nova:show_success` in minion config/pillar. show_compliance Whether to show compliance as a percentage (successful checks divided by total checks). Defaults to True. Configurable via `hubblestack:nova:show_compliance` in minion config/pillar. show_profile DEPRECATED debug Whether to log additional information to help debug nova. Defaults to False. Configurable via `hubblestack:nova:debug` in minion config/pillar. CLI Examples: .. code-block:: bash salt '*' hubble.top salt '*' hubble.top foo/bar/top.nova salt '*' hubble.top foo/bar.nova verbose=True ''' if __salt__['config.get']('hubblestack:nova:autoload', True): load() if not __nova__: return False, 'No nova modules/data have been loaded.' if verbose is None: verbose = __salt__['config.get']('hubblestack:nova:verbose', False) if show_success is None: show_success = __salt__['config.get']('hubblestack:nova:show_success', True) if show_compliance is None: show_compliance = __salt__['config.get']('hubblestack:nova:show_compliance', True) if show_profile is not None: log.warning( 'Keyword argument \'show_profile\' is no longer supported' ) if debug is None: debug = __salt__['config.get']('hubblestack:nova:debug', False) results = {} # Get a list of yaml to run top_data = _get_top_data(topfile) # Will be a combination of strings and single-item dicts. The strings # have no tag filters, so we'll treat them as tag filter '*'. If we sort # all the data by tag filter we can batch where possible under the same # tag. data_by_tag = {} for data in top_data: if isinstance(data, basestring): if '*' not in data_by_tag: data_by_tag['*'] = [] data_by_tag['*'].append(data) elif isinstance(data, dict): for key, tag in data.iteritems(): if tag not in data_by_tag: data_by_tag[tag] = [] data_by_tag[tag].append(key) else: if 'Errors' not in results: results['Errors'] = {} error_log = 'topfile malformed, list entries must be strings or '\ 'dicts: {0} | {1}'.format(data, type(data)) results['Errors'][topfile] = {'error': error_log} log.error(error_log) continue if not data_by_tag: return results # Run the audits for tag, data in data_by_tag.iteritems(): ret = audit(configs=data, tags=tag, verbose=verbose, show_success=True, show_compliance=False, called_from_top=True, labels=labels) # Merge in the results for key, val in ret.iteritems(): if key not in results: results[key] = [] results[key].extend(val) if show_compliance: compliance = _calculate_compliance(results) if compliance: results['Compliance'] = compliance for key in results.keys(): if not results[key]: results.pop(key) if not results: results['Messages'] = 'No audits matched this host in the specified profiles.' if not show_success and 'Success' in results: results.pop('Success') return results
def sync(clean=False): ''' Sync the nova audit modules and profiles from the saltstack fileserver. The modules should be stored in the salt fileserver. By default nova will search the base environment for a top level ``hubblestack_nova`` directory, unless otherwise specified via pillar or minion config (``hubblestack:nova:module_dir``) The profiles should be stored in the salt fileserver. By default nova will search the base environment for a top level ``hubblestack_nova_profiles`` directory, unless otherwise specified via pillar or minion config (``hubblestack:nova:profile_dir``) Modules and profiles will be cached in the normal minion cachedir Returns a boolean representing success NOTE: This function will optionally clean out existing files at the cached location, as cp.cache_dir doesn't clean out old files. Pass ``clean=True`` to enable this behavior CLI Examples: .. code-block:: bash salt '*' nova.sync salt '*' nova.sync saltenv=hubble ''' log.debug('syncing nova modules') nova_profile_dir = __salt__['config.get']('hubblestack:nova:profile_dir', 'salt://hubblestack_nova_profiles') nova_module_dir, cached_profile_dir = _hubble_dir() saltenv = __salt__['config.get']('hubblestack:nova:saltenv', 'base') # Clean previously synced files if clean: __salt__['file.remove'](cached_profile_dir) synced = [] # Support optional salt:// in config if 'salt://' in nova_profile_dir: path = nova_profile_dir _, _, nova_profile_dir = nova_profile_dir.partition('salt://') else: path = 'salt://{0}'.format(nova_profile_dir) # Sync the files cached = __salt__['cp.cache_dir'](path, saltenv=saltenv) if cached and isinstance(cached, list): # Success! Trim the paths cachedir = os.path.dirname(cached_profile_dir) ret = [relative.partition(cachedir)[2] for relative in cached] synced.extend(ret) else: if isinstance(cached, list): # Nothing was found synced.extend(cached) else: # Something went wrong, there's likely a stacktrace in the output # of cache_dir raise CommandExecutionError('An error occurred while syncing: {0}' .format(cached)) return synced def load(): ''' Load the synced audit modules. ''' if __salt__['config.get']('hubblestack:nova:autosync', True): sync() for nova_dir in _hubble_dir(): if not os.path.isdir(nova_dir): return False, 'No synced nova modules/profiles found' log.debug('loading nova modules') global __nova__ __nova__ = NovaLazyLoader(_hubble_dir(), __opts__, __grains__, __pillar__, __salt__) ret = {'loaded': __nova__._dict.keys(), 'missing': __nova__.missing_modules, 'data': __nova__.__data__.keys(), 'missing_data': __nova__.__missing_data__} return ret def version(): ''' Report the version of this module ''' return __version__ def _hubble_dir(): ''' Generate the local minion directories to which nova modules and profiles are synced Returns a tuple of two paths, the first for nova modules, the second for nova profiles ''' nova_profile_dir = __salt__['config.get']('hubblestack:nova:profile_dir', 'salt://hubblestack_nova_profiles') nova_module_dir = os.path.join(__opts__['install_dir'], 'files', 'hubblestack_nova') # Support optional salt:// in config if 'salt://' in nova_profile_dir: _, _, nova_profile_dir = nova_profile_dir.partition('salt://') saltenv = __salt__['config.get']('hubblestack:nova:saltenv', 'base') cachedir = os.path.join(__opts__.get('cachedir'), 'files', saltenv, nova_profile_dir) dirs = [nova_module_dir, cachedir] return tuple(dirs) def _calculate_compliance(results): ''' Calculate compliance numbers given the results of audits ''' success = len(results.get('Success', [])) failure = len(results.get('Failure', [])) control = len(results.get('Controlled', [])) total_audits = success + failure + control if total_audits: compliance = float(success + control) / total_audits compliance = int(compliance * 100) compliance = '{0}%'.format(compliance) return compliance return None def _get_top_data(topfile): ''' Helper method to retrieve and parse the nova topfile ''' topfile = os.path.join(_hubble_dir()[1], topfile) try: with open(topfile) as handle: topdata = yaml.safe_load(handle) except Exception as e: raise CommandExecutionError('Could not load topfile: {0}'.format(e)) if not isinstance(topdata, dict) or 'nova' not in topdata or \ not(isinstance(topdata['nova'], dict)): raise CommandExecutionError('Nova topfile not formatted correctly') topdata = topdata['nova'] ret = [] for match, data in topdata.iteritems(): if __salt__['match.compound'](match): ret.extend(data) return ret