log when runner is behind
This commit is contained in:
@@ -213,6 +213,7 @@ class BlockStateRunner(BlockProgressor):
|
|||||||
|
|
||||||
|
|
||||||
async def worker(self):
|
async def worker(self):
|
||||||
|
was_behind = False
|
||||||
try:
|
try:
|
||||||
log.info(f'Runner started')
|
log.info(f'Runner started')
|
||||||
w3 = current_w3.get()
|
w3 = current_w3.get()
|
||||||
@@ -263,6 +264,14 @@ class BlockStateRunner(BlockProgressor):
|
|||||||
else:
|
else:
|
||||||
log.info(f'Reverting {fork.branch} due to {e}') # runtime exception like SystemExit
|
log.info(f'Reverting {fork.branch} due to {e}') # runtime exception like SystemExit
|
||||||
self.state.remove_branch(fork.branch)
|
self.state.remove_branch(fork.branch)
|
||||||
|
processed_block = await get_block(fork.head)
|
||||||
|
behind = timestamp() - processed_block.timestamp
|
||||||
|
if behind > 2:
|
||||||
|
# todo pagerduty alerts if too far behind
|
||||||
|
log.info(f'Runner is {timedelta(seconds=behind)} behind')
|
||||||
|
was_behind = True
|
||||||
|
elif was_behind:
|
||||||
|
log.info('Runner has caught up')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
fatal('Unhandled exception in runner worker', exception=e)
|
fatal('Unhandled exception in runner worker', exception=e)
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
Reference in New Issue
Block a user