diff --git a/src/dexorder/runner.py b/src/dexorder/runner.py index 1e9a352..21c9ee1 100644 --- a/src/dexorder/runner.py +++ b/src/dexorder/runner.py @@ -213,6 +213,7 @@ class BlockStateRunner(BlockProgressor): async def worker(self): + was_behind = False try: log.info(f'Runner started') w3 = current_w3.get() @@ -263,6 +264,14 @@ class BlockStateRunner(BlockProgressor): else: log.info(f'Reverting {fork.branch} due to {e}') # runtime exception like SystemExit self.state.remove_branch(fork.branch) + processed_block = await get_block(fork.head) + behind = timestamp() - processed_block.timestamp + if behind > 2: + # todo pagerduty alerts if too far behind + log.info(f'Runner is {timedelta(seconds=behind)} behind') + was_behind = True + elif was_behind: + log.info('Runner has caught up') except Exception as e: fatal('Unhandled exception in runner worker', exception=e) finally: