diff --git a/shadowsocks/eventloop.py b/shadowsocks/eventloop.py
index 2b2459f..304b229 100644
--- a/shadowsocks/eventloop.py
+++ b/shadowsocks/eventloop.py
@@ -232,9 +232,10 @@ class EventLoop(object):
                     logging.error(e)
                     import traceback
                     traceback.print_exc()
-            for handler in self._handlers_to_remove:
-                self._handlers.remove(handler)
-            self._handlers_to_remove = []
+            if self._handlers_to_remove:
+                for handler in self._handlers_to_remove:
+                    self._handlers.remove(handler)
+                self._handlers_to_remove = []
             self._iterating = False