70 lines
2.1 KiB
Python
70 lines
2.1 KiB
Python
import logging
|
|
import socket
|
|
|
|
import pdpyras
|
|
|
|
from dexorder import NARG, config
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def alert(title, message, dedup_key=NARG, log_level=logging.ERROR, do_log=True):
|
|
if dedup_key is NARG:
|
|
dedup_key = str(hash(title))
|
|
if do_log:
|
|
msg = f'{title}: {message}'
|
|
log.log(log_level, msg) # if log_level=CRITICAL for example, make sure this does not re-alert!
|
|
alert_pagerduty(title, message, dedup_key, log_level)
|
|
|
|
|
|
def warningAlert(title, message, dedup_key=NARG, log_level=logging.WARNING):
|
|
return alert(title, message, dedup_key, log_level)
|
|
|
|
|
|
pagerduty_session = None
|
|
hostname = None
|
|
|
|
def alert_pagerduty(title, message, dedup_key, log_level):
|
|
if not config.pagerduty:
|
|
return
|
|
# noinspection PyBroadException
|
|
try:
|
|
global pagerduty_session
|
|
global hostname
|
|
if pagerduty_session is None:
|
|
pagerduty_session = pdpyras.EventsAPISession(config.pagerduty)
|
|
hostname = socket.gethostname()
|
|
sev = 'error' if log_level >= logging.ERROR else 'warning'
|
|
pagerduty_session.trigger(title, hostname, severity=sev, custom_details={'message': message}, dedup_key=dedup_key)
|
|
except Exception:
|
|
log.warning('Could not notify PagerDuty!', exc_info=True)
|
|
|
|
|
|
class AlertHandler(logging.Handler):
|
|
|
|
def __init__(self, level=logging.NOTSET):
|
|
super().__init__(level)
|
|
self.paused = False
|
|
|
|
def emit(self, record):
|
|
if not self.paused:
|
|
msg = self.format(record)
|
|
if msg is not None and type(msg) is str:
|
|
msg = msg.strip()
|
|
msg_last_line = msg.split('\n')[-1]
|
|
if msg_last_line:
|
|
msg_last_line = ': ' + msg_last_line[:100]
|
|
else:
|
|
msg_last_line = ''
|
|
self.paused = True
|
|
try:
|
|
alert(record.levelname + msg_last_line, msg, log_level=record.levelno, do_log=False)
|
|
finally:
|
|
self.paused = False
|
|
|
|
|
|
def init_alerts():
|
|
if config.pagerduty:
|
|
logging.getLogger('dexorder').addHandler(AlertHandler(logging.WARNING))
|
|
|