import logging import socket import pdpyras from dexorder import NARG, config log = logging.getLogger(__name__) def alert(title, message, dedup_key=NARG, log_level=logging.ERROR, do_log=True): if dedup_key is NARG: dedup_key = str(hash(title)) if do_log: msg = f'{title}: {message}' log.log(log_level, msg) # if log_level=CRITICAL for example, make sure this does not re-alert! alert_pagerduty(title, message, dedup_key, log_level) def warningAlert(title, message, dedup_key=NARG, log_level=logging.WARNING): return alert(title, message, dedup_key, log_level) pagerduty_session = None hostname = None def alert_pagerduty(title, message, dedup_key, log_level): if not config.pagerduty: return # noinspection PyBroadException try: global pagerduty_session global hostname if pagerduty_session is None: pagerduty_session = pdpyras.EventsAPISession(config.pagerduty) hostname = socket.gethostname() sev = 'error' if log_level >= logging.ERROR else 'warning' pagerduty_session.trigger(title, hostname, severity=sev, custom_details={'message': message}, dedup_key=dedup_key) except Exception: log.warning('Could not notify PagerDuty!', exc_info=True) class AlertHandler(logging.Handler): def __init__(self, level=logging.NOTSET): super().__init__(level) self.paused = False def emit(self, record): if not self.paused: msg = self.format(record) if msg is not None and type(msg) is str: msg = msg.strip() msg_last_line = msg.split('\n')[-1] if msg_last_line: msg_last_line = ': ' + msg_last_line[:100] else: msg_last_line = '' self.paused = True try: alert(record.levelname + msg_last_line, msg, log_level=record.levelno, do_log=False) finally: self.paused = False def init_alerts(): if config.pagerduty: logging.getLogger('dexorder').addHandler(AlertHandler(logging.WARNING))