Skip to content

Instantly share code, notes, and snippets.

@alias454

alias454/inventory.yml

Last active October 25, 2021 17:34
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alias454/02598a594fe609c4b8f650772a9c8790 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alias454/02598a594fe609c4b8f650772a9c8790 to your computer and use it in GitHub Desktop.
Download ZIP
WIP splunk patch and reboot fabric script
Raw
inventory.yml
---
# list of node groups
service:
splunk:
config:
verify_tls: False
splunk_cm_api_url: 'https://splunk-cluster-master01.domain.tld:8089'
splunk_status_check_path: 'services/cluster/master/status?output_mode=json'
splunk_set_mm_path: 'services/cluster/master/control/default/maintenance?output_mode=json'
roles:
lm:
- "splunk-license-master01.domain.tld"
ds:
- "splunk-deployment-server01.domain.tld"
cm:
- "splunk-cluster-master01.domain.tld"
shc:
- "splunk-search-head01.domain.tld"
- "splunk-search-head02.domain.tld"
- "splunk-search-head03.domain.tld"
idx:
- "splunk-indexer01.domain.tld"
- "splunk-indexer02.domain.tld"
- "splunk-indexer03.domain.tld"
Raw
patch_and_reboot.py
import requests
import urllib3
import yaml
from requests.auth import HTTPBasicAuth
from yaml.loader import SafeLoader
from time import sleep
from fabric import Connection, Config
from paramiko.ssh_exception import NoValidConnectionsError
# Read inventory and config data from file
inv_file_path = 'inventory.yml'
with open(inv_file_path) as inv_file:
inv_data = yaml.load(inv_file, Loader=SafeLoader)
roles = inv_data['service']['splunk']['roles']
config = inv_data['service']['splunk']['config']
# Disable bad cert warnings when config['verify_tls'] is false
verify_tls = config['verify_tls']
if not verify_tls:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Account credentials
username = ''
password = ''
rest_user = ''
rest_user_pass = ''
def connect(host, user, passwd=''):
"""
Setup SSH connection to a host
:param host: Hostname of an instance to connect to
:param user: Username to use when connecting
:param passwd: Password to use for username
:return: A connection object
"""
override = Config(overrides={'sudo': {'password': passwd}})
ret = Connection(host=host, user=user, config=override, connect_kwargs={"password": passwd})
return ret
def join_api_url(api_base_url, api_path):
"""
Simple join of a base url and an api path
:param api_base_url: String representing the base url to use when connecting
:param api_path: String representing the api path
:return: A joined url with path
"""
return api_base_url + '/' + api_path
def check_status(con):
"""
Check status by verifying Splunkd is running using sudo
:param con: Connection obj
:return: True if is running else false
"""
try:
status = con.sudo('su - splunk -c "/opt/splunk/bin/splunk status"', hide=True)
if 'is running' in status.stdout:
return True
else:
return False
except (ConnectionError, NoValidConnectionsError):
return False
def patch_os(con):
"""
Run the command to patch the OS using sudo.
:param con: Connection obj
:return: Result from command
"""
try:
ret = con.sudo('yum update -y', hide=True)
return ret
except (ConnectionError, NoValidConnectionsError):
ret = 'Connection error aborting'
return ret
def cmd_wait(con, run_cmd):
"""
Command will be retried until success unless range value is reached
:param con: Connection obj
:param run_cmd: Command to run
:return: Result from command
"""
# May take up to 5 minutes
sleep(5)
ret = 'Command took too long to finish'
for _ in range(25):
try:
ret = con.run(run_cmd, hide=True)
break
except (ConnectionError, NoValidConnectionsError):
sleep(10)
return ret
def check_maintenance_mode(api_url, user, user_pass, verify=False):
"""
Check if maintenance mode is enabled, or disabled based on the value set for maintenance_mode
:param api_url: String representing the base url to use when connecting
:param user: Rest User to use when connecting
:param user_pass: Password to use for Rest User
:param verify: Verify TLS
:return: Enabled, disabled, or unknown depending on requests value
"""
try:
ret = requests.get(api_url, auth=HTTPBasicAuth(user, user_pass), verify=verify)
if ret.json()['entry'][0]['content']['maintenance_mode']:
return 'enabled'
else:
return 'disabled'
except (ConnectionError, NoValidConnectionsError):
return 'unknown'
def set_maintenance_mode(api_set_mm, user, user_pass, data, verify=False):
"""
Set maintenance mode status based on mode passed in as data
:param api_set_mm: String representing the base url to use when connecting
:param user: Rest User to use when connecting
:param user_pass: Password to use for Rest User
:param data: Mode value is True or False
:param verify: Verify TLS
:return: Result from API
"""
try:
ret = requests.post(api_set_mm, auth=HTTPBasicAuth(user, user_pass), data=data, verify=verify)
return ret
except (ConnectionError, NoValidConnectionsError):
ret = 'Connection error aborting'
return ret
# Join API url and path for maintenance status check
check_status_url = join_api_url(config['splunk_cm_api_url'], config['splunk_status_check_path'])
# False is the expected value but we return disabled
if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) == 'disabled':
# Perform checks to verify ready to start
for role in roles.keys():
for hostname in roles[role]:
# Create a new session and check Splunkd status
session = connect(hostname, username, password)
if check_status(session):
print('[OK] Splunkd is running: ' + hostname)
else:
print('[FAIL] Status Check: ' + hostname)
# Close session and exit
session.close()
exit(1)
# Close session and continue
session.close()
else:
# Maintenance mode is already enabled.
# We should find out why and try again
print('[FAIL] Maintenance mode already enabled')
exit(1)
print('[OK] All status checks passed... Proceeding')
# Join API url and path for setting maintenance mode value
splunk_api_set_mm = join_api_url(config['splunk_cm_api_url'], config['splunk_set_mm_path'])
# Status checks have passed, do the updates
for role in roles.keys():
# Set maintenance mode status to true
# Only needs to be done for the indexers
if role == 'idx':
rest_data = {'mode': 'true'}
res = set_maintenance_mode(splunk_api_set_mm, rest_user, rest_user_pass, rest_data, verify_tls)
if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) != 'enabled':
print('[FAIL] Maintenance mode not enabled:')
exit(1)
# Iterate through list of hosts assigned to this role
for hostname in roles[role]:
# Create session
session = connect(hostname, username, password)
# Update and restart host if patches applied
is_updated = patch_os(session)
if 'Complete!' in is_updated.stdout and 'Nothing to do' not in is_updated.stdout:
if role == 'idx':
# Take splunk offline
pass
is_rebooted = session.sudo('nohup sudo -b bash -c "sleep 5 && reboot"', hide=True)
if is_rebooted.return_code == 0:
is_running = cmd_wait(session, 'systemctl status Splunkd.service')
if is_running.return_code == 0:
print('[OK] Splunkd is running: ' + hostname)
print('[OK] Reboot successful: ' + hostname)
else:
print('[FAIL] Status Check: ' + hostname)
else:
print('[OK] No updates: ' + hostname)
# Close session
session.close()
# Set maintenance mode status to false
# Only needs to be done for the indexers
if role == 'idx':
rest_data = {'mode': 'false'}
res = set_maintenance_mode(splunk_api_set_mm, rest_user, rest_user_pass, rest_data, verify_tls)
if check_maintenance_mode(check_status_url, rest_user, rest_user_pass, verify_tls) != 'disabled':
print('[FAIL] Maintenance mode not disabled:')
exit(1)
print('[OK] All hosts finished. Complete!')
Raw
requirements.txt
fabric==2.6.0
invoke==1.6.0
PyYAML==6.0
requests==2.26.0
urllib3~=1.26.7
paramiko~=2.8.0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment