Skip to content
This repository was archived by the owner on Feb 9, 2024. It is now read-only.

Do not exit on DataDog API errors #27

Open
AMeng wants to merge 15 commits into
base: master
Choose a base branch
from
Open

Do not exit on DataDog API errors #27

Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
26c3a71
Adds exception handling around the interaction with the Datadog API.
Nov 27, 2017
b810838
Will raise a final exception to ensure process still returns a non-ze…
Nov 27, 2017
9ff4959
More idiomatic string interpolation
Nov 27, 2017
a34febd
Ignore wildcard tags
AMeng Dec 19, 2017
d8caab9
Merge pull request #1 from fullcontact/ignore-wildcard-tags
AMeng Dec 19, 2017
2978a10
following recommendation from datadog support
Aug 31, 2018
aa77505
Merge pull request #2 from fullcontact/alee/datadog_troubleshoot
Aug 31, 2018
464ad08
python3 compatible
Oct 24, 2018
374506c
Merge pull request #3 from fullcontact/alee/python3_compliance
Oct 24, 2018
e5dd69f
ignore include_tags option and make sure to always use yaml block style
jeremyplichtafc Jan 24, 2019
170e4e2
Merge pull request #4 from fullcontact/jplichta/bug/fix-formatting
Jan 24, 2019
05220a3
add defaults instead of removing them so diff works like it should
Jun 26, 2019
d9f403c
Merge pull request #5 from fullcontact/tsnyder/appendDefaults
tony-snyder-fc Jun 26, 2019
dfa9bc9
add restricted_roles as a field to ignore for rules creation and diffing
Apr 2, 2020
7dbaa58
Merge pull request #6 from fullcontact/tsnyder/restrictedRolesRemoval
tony-snyder-fc Apr 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 63 additions & 40 deletions dogpush/dogpush.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@

import datadog
import datadog.api
from datadog.api.exceptions import ApiError
import pytz
import yaml

import bcolors


PROGNAME = 'dogpush'
ERRORS_LIST = []


class DogPushException(Exception):
Expand Down Expand Up @@ -63,10 +65,11 @@ def _load_config(config_file):
IGNORE_FIELDS = ['created_at', 'created', 'modified', 'creator',
'org_id', 'overall_state', 'id', 'deleted',
'matching_downtimes', 'overall_state_modified',
'restricted_roles',
# dogpush specific:
'mute_when', 'team', 'severity']

IGNORE_OPTIONS = ['silenced']
IGNORE_OPTIONS = ['silenced', 'include_tags']

# Datadog fields that we do not store in our monitor rules if they have the
# default value.
Expand All @@ -90,13 +93,13 @@ def _load_config(config_file):

def _pretty_yaml(d):
return re.sub('^-', '\n-',
yaml.dump(d, width=CONFIG['dogpush']['yaml_width']), flags=re.M)
yaml.dump(d, width=CONFIG['dogpush']['yaml_width'], default_flow_style=False), flags=re.M)


# Transform a monitor to a canonical form by removing defaults
def _canonical_monitor(original, default_team=None, **kwargs):
m = copy.deepcopy(original)
if 'tags' in m and not m['tags']:
if 'tags' in m and (m['tags'] == ['*'] or not m['tags']):
del m['tags']
for field in IGNORE_FIELDS:
m.pop(field, None)
Expand All @@ -108,8 +111,8 @@ def _canonical_monitor(original, default_team=None, **kwargs):
if m.get('options', {}).get(field) == value:
del m['options'][field]
for (field, value) in CONFIG['default_rules'].items():
if m.get(field) == value:
del m[field]
if field not in m:
m[field] = value
# If options is {'thresholds': {'critical': x}}, then it is redundant.
if not m.get('options'):
m.pop('options', None)
Expand Down Expand Up @@ -139,7 +142,7 @@ def _canonical_monitor(original, default_team=None, **kwargs):


def get_datadog_monitors():
monitors = datadog.api.Monitor.get_all(with_downtimes="true")
monitors = datadog.api.Monitor.get_all(with_downtimes="false")
if CONFIG['dogpush']['ignore_prefix'] is not None:
monitors = [
m for m in monitors
Expand Down Expand Up @@ -226,9 +229,9 @@ def _is_changed(local, remote):
def command_init(args):
remote_monitors = [m['obj'] for m in get_datadog_monitors().values()]
monitors = {'alerts': remote_monitors}
print '# team: TEAMNAME'
print
print _pretty_yaml(monitors)
print('# team: TEAMNAME')
print()
print(_pretty_yaml(monitors))


def command_push(args):
Expand All @@ -237,27 +240,40 @@ def command_push(args):

only_local = set(local_monitors.keys()) - set(remote_monitors.keys())
if only_local:
print "Pushing %d new monitors." % len(only_local)
print("Pushing %d new monitors." % len(only_local))
for name in only_local:
datadog.api.Monitor.create(**_prepare_monitor(local_monitors[name]))
try:
datadog.api.Monitor.create(**_prepare_monitor(local_monitors[name]))
except ApiError as ae:
ERRORS_LIST.append(ae)
print("The monitor, '{0}', was not created due to a Datadog API error.\n({1})".format(name, ae.message))

common_names = set(local_monitors.keys()) & set(remote_monitors.keys())
changed = [name for name in common_names
if _is_changed(local_monitors[name], remote_monitors[name])]
if changed:
print "Updating %d modified monitors." % len(changed)
print("Updating %d modified monitors." % len(changed))
for name in changed:
datadog.api.Monitor.update(
remote_monitors[name]['id'],
**_prepare_monitor(local_monitors[name]))

try:
datadog.api.Monitor.update(
remote_monitors[name]['id'],
**_prepare_monitor(local_monitors[name]))
except ApiError as ae:
ERRORS_LIST.append(ae)
print("The monitor, '{0}', was not updated due to a Datadog API error.\n({1})".format(name, ae.message))
if args.delete_untracked:
remote_monitors = get_datadog_monitors()
untracked = set(remote_monitors.keys()) - set(local_monitors.keys())
if untracked:
print "Deleting %d untracked monitors." % len(untracked)
print("Deleting %d untracked monitors." % len(untracked))
for monitor in untracked:
datadog.api.Monitor.delete(remote_monitors[monitor]['id'])
try:
datadog.api.Monitor.delete(remote_monitors[monitor]['id'])
except ApiError as ae:
ERRORS_LIST.append(ae)
print("The monitor, '{0}', was not deleted due to a Datadog API error.\n({1})".format(
monitor,
ae.message))


def _should_mute(expr, tz, now):
Expand Down Expand Up @@ -302,14 +318,18 @@ def command_mute(args):
if monitor['mute_when'] and remote_monitors.has_key(monitor['name']):
remote = remote_monitors[monitor['name']]
if remote['is_silenced']:
print "Alert '%s' is already muted. Skipping." % monitor['name']
print("Alert '%s' is already muted. Skipping." % monitor['name'])
continue
mute_until = mute_tags[monitor['mute_when']]
if mute_until:
id = remote['id']
datadog.api.Monitor.mute(id, end=mute_until['timestamp'])
print "Muting alert '%s' until %s" % (monitor['name'],
mute_until['datetime'])
try:
datadog.api.Monitor.mute(id, end=mute_until['timestamp'])
print("Muting alert '%s' until %s" % (monitor['name'],
mute_until['datetime']))
except ApiError as ae:
ERRORS_LIST.append(ae)
print("The monitor, '{0}', was not muted due to a Datadog API error.\n({0})".format(id, ae.message))


def command_diff(args):
Expand All @@ -325,21 +345,21 @@ def command_diff(args):

if only_local:
sys.stdout.write(bcolors.WARNING)
print '---------------------------------------------------------'
print ' NEW MONITORS. These monitors are currently missing in'
print ' datadog and can be pushed using "%s push"' % PROGNAME
print '---------------------------------------------------------'
print('---------------------------------------------------------')
print(' NEW MONITORS. These monitors are currently missing in')
print(' datadog and can be pushed using "%s push"' % PROGNAME)
print('---------------------------------------------------------')
sys.stdout.write(bcolors.ENDC)
monitors = [local_monitors[name]['obj'] for name in only_local]
print _pretty_yaml(monitors)
print(_pretty_yaml(monitors))
if changed:
sys.stdout.write(bcolors.WARNING)
print '---------------------------------------------------------'
print ' TO BE UPDATED. These monitors exist in datadog, but are'
print ' different than the local version. Use "%s push"' % PROGNAME
print ' to push them to datadog.'
print '---------------------------------------------------------'
print
print('---------------------------------------------------------')
print(' TO BE UPDATED. These monitors exist in datadog, but are')
print(' different than the local version. Use "%s push"' % PROGNAME)
print(' to push them to datadog.')
print('---------------------------------------------------------')
print()
sys.stdout.write(bcolors.ENDC)
for name in changed:
remote_name = 'datadog:%s' % name
Expand All @@ -358,16 +378,16 @@ def command_diff(args):
sys.stdout.write(line)
if only_remote and not args.ignore_untracked:
sys.stdout.write(bcolors.WARNING)
print '------------------------------------------------------------'
print ' UNTRACKED MONITORS. These monitors are only in datadog '
print ' and needed to be MANUALLY added to a local file or removed '
print ' from datadog. '
print '------------------------------------------------------------'
print('------------------------------------------------------------')
print(' UNTRACKED MONITORS. These monitors are only in datadog ')
print(' and needed to be MANUALLY added to a local file or removed ')
print(' from datadog. ')
print('------------------------------------------------------------')
sys.stdout.write(bcolors.ENDC)
monitors = [remote_monitors[name]['obj'] for name in only_remote]
print _pretty_yaml(monitors)
print(_pretty_yaml(monitors))
sys.stdout.write(bcolors.FAIL)
print "*** FAILED *** Untracked monitors found."
print("*** FAILED *** Untracked monitors found.")
sys.stdout.write(bcolors.ENDC)
if args.exit_status and any((only_local, changed, only_remote and not args.ignore_untracked)):
sys.exit(1)
Expand Down Expand Up @@ -426,6 +446,9 @@ def command_diff(args):
def main():
datadog.initialize(**CONFIG['datadog'])
args.command(args)
if len(ERRORS_LIST) > 0:
raise DogPushException('One or more exceptions were caught while running dogpush. '
'Please review standard output for more information.')


if __name__ == '__main__':
Expand Down