Merge branch '218-handle-exception-in-loop-thread' into 'main'

Resolve "Handle exception in loop thread"

Closes #218

See merge request inject/backend!220

running_exercise/lib/loop_thread.py

@@ -32,48 +32,71 @@ class LoopThread(Thread):
         self.__finished.set()
 
     def run(self):
-        SubscriptionHandler.broadcast_exercise_loop(self.updater.exercise, True)
-        self.updater.update(self.elapsed_s)
-        SubscriptionHandler.broadcast_exercises(
-            self.updater.exercise, ExerciseEventTypeEnum.modify()
-        )
-        wait_time: float = self.__interval_s
-        while (
-            self.elapsed_s <= self.__total_time_s
-            and not self.__finished.wait(wait_time)
-        ):
-            update_time = self.updater.update(self.elapsed_s)
-            wait_time = (
-                self.__interval_s - update_time
-                if update_time < self.__interval_s
-                else 0
+        try:
+            SubscriptionHandler.broadcast_exercise_loop(
+                self.updater.exercise, True
             )
-            # this is necessary to prevent infinite length loops with 0 interval
-            # I wouldn't expect this during any real deployment, however it might
-            # be useful during testing for max speed update times
-            if self.__interval_s == 0:
-                self.elapsed_s += 1
-            else:
-                self.elapsed_s += self.__interval_s
-            logger.info(
-                f"update loop for exercise id: {self.updater.exercise.id} elapsed time: {self.elapsed_s}s"
+            self.updater.update(self.elapsed_s)
+            SubscriptionHandler.broadcast_exercises(
+                self.updater.exercise, ExerciseEventTypeEnum.modify()
             )
+            wait_time: float = self.__interval_s
+            while (
+                self.elapsed_s <= self.__total_time_s
+                and not self.__finished.wait(wait_time)
+            ):
+                update_time = self.updater.update(self.elapsed_s)
+                wait_time = (
+                    self.__interval_s - update_time
+                    if update_time < self.__interval_s
+                    else 0
+                )
+                # this is necessary to prevent infinite length loops with 0 interval
+                # I wouldn't expect this during any real deployment, however it might
+                # be useful during testing for max speed update times
+                if self.__interval_s == 0:
+                    self.elapsed_s += 1
+                else:
+                    self.elapsed_s += self.__interval_s
+                logger.info(
+                    f"update loop for exercise id: {self.updater.exercise.id} elapsed time: {self.elapsed_s}s"
+                )
 
-        SubscriptionHandler.broadcast_exercise_loop(
-            self.updater.exercise, False
-        )
-        self.__finished.set()
+            SubscriptionHandler.broadcast_exercise_loop(
+                self.updater.exercise, False
+            )
+            self.__finished.set()
 
-        if self.elapsed_s >= self.__total_time_s:
-            self.updater.finish()
-            logger.info(
-                f"finishing exercise loop for exercise id: {self.updater.exercise.id}"
+            if self.elapsed_s >= self.__total_time_s:
+                self.updater.finish()
+                logger.info(
+                    f"finishing exercise loop for exercise id: {self.updater.exercise.id}"
+                )
+            else:
+                self.updater.stop(self.elapsed_s)
+            SubscriptionHandler.broadcast_exercises(
+                self.updater.exercise, ExerciseEventTypeEnum.modify()
+            )
+        except Exception as ex:
+            logger.error(
+                f"loop thread exception {type(ex).__name__}: {str(ex)}"
             )
-        else:
-            self.updater.stop(self.elapsed_s)
-        SubscriptionHandler.broadcast_exercises(
-            self.updater.exercise, ExerciseEventTypeEnum.modify()
-        )
+            for i in range(10):
+                try:
+                    self.updater.stop(self.elapsed_s - self.__interval_s)
+                except:
+                    logger.error(
+                        f"failed to stop exercise: {self.updater.exercise.id}"
+                    )
+                else:
+                    logger.info(
+                        f"stopped exercise: {self.updater.exercise.id} on the {i}th try"
+                    )
+                    break
+            else:
+                logger.error(
+                    f"failed to stop exercise: {self.updater.exercise.id} 10 times, still running"
+                )
 
     def remaining_time_s(self) -> int:
         return self.__total_time_s - self.elapsed_s

ttxbackend/settings.py

@@ -100,6 +100,9 @@ DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.sqlite3",
         "NAME": os.path.join(BASE_DIR, DATA_STORAGE, "db.sqlite3"),
+        "OPTIONS": {
+            "timeout": 20,
+        },
     }
 }