_reloader.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. import os
  2. import sys
  3. import time
  4. import subprocess
  5. import threading
  6. from itertools import chain
  7. from werkzeug._internal import _log
  8. from werkzeug._compat import PY2, iteritems, text_type
  9. def _iter_module_files():
  10. """This iterates over all relevant Python files. It goes through all
  11. loaded files from modules, all files in folders of already loaded modules
  12. as well as all files reachable through a package.
  13. """
  14. # The list call is necessary on Python 3 in case the module
  15. # dictionary modifies during iteration.
  16. for module in list(sys.modules.values()):
  17. if module is None:
  18. continue
  19. filename = getattr(module, '__file__', None)
  20. if filename:
  21. old = None
  22. while not os.path.isfile(filename):
  23. old = filename
  24. filename = os.path.dirname(filename)
  25. if filename == old:
  26. break
  27. else:
  28. if filename[-4:] in ('.pyc', '.pyo'):
  29. filename = filename[:-1]
  30. yield filename
  31. def _find_observable_paths(extra_files=None):
  32. """Finds all paths that should be observed."""
  33. rv = set(os.path.abspath(x) for x in sys.path)
  34. for filename in extra_files or ():
  35. rv.add(os.path.dirname(os.path.abspath(filename)))
  36. for module in list(sys.modules.values()):
  37. fn = getattr(module, '__file__', None)
  38. if fn is None:
  39. continue
  40. fn = os.path.abspath(fn)
  41. rv.add(os.path.dirname(fn))
  42. return _find_common_roots(rv)
  43. def _find_common_roots(paths):
  44. """Out of some paths it finds the common roots that need monitoring."""
  45. paths = [x.split(os.path.sep) for x in paths]
  46. root = {}
  47. for chunks in sorted(paths, key=len, reverse=True):
  48. node = root
  49. for chunk in chunks:
  50. node = node.setdefault(chunk, {})
  51. node.clear()
  52. rv = set()
  53. def _walk(node, path):
  54. for prefix, child in iteritems(node):
  55. _walk(child, path + (prefix,))
  56. if not node:
  57. rv.add('/'.join(path))
  58. _walk(root, ())
  59. return rv
  60. class ReloaderLoop(object):
  61. name = None
  62. # monkeypatched by testsuite. wrapping with `staticmethod` is required in
  63. # case time.sleep has been replaced by a non-c function (e.g. by
  64. # `eventlet.monkey_patch`) before we get here
  65. _sleep = staticmethod(time.sleep)
  66. def __init__(self, extra_files=None, interval=1):
  67. self.extra_files = set(os.path.abspath(x)
  68. for x in extra_files or ())
  69. self.interval = interval
  70. def run(self):
  71. pass
  72. def restart_with_reloader(self):
  73. """Spawn a new Python interpreter with the same arguments as this one,
  74. but running the reloader thread.
  75. """
  76. while 1:
  77. _log('info', ' * Restarting with %s' % self.name)
  78. args = [sys.executable] + sys.argv
  79. new_environ = os.environ.copy()
  80. new_environ['WERKZEUG_RUN_MAIN'] = 'true'
  81. # a weird bug on windows. sometimes unicode strings end up in the
  82. # environment and subprocess.call does not like this, encode them
  83. # to latin1 and continue.
  84. if os.name == 'nt' and PY2:
  85. for key, value in iteritems(new_environ):
  86. if isinstance(value, text_type):
  87. new_environ[key] = value.encode('iso-8859-1')
  88. exit_code = subprocess.call(args, env=new_environ)
  89. if exit_code != 3:
  90. return exit_code
  91. def trigger_reload(self, filename):
  92. filename = os.path.abspath(filename)
  93. _log('info', ' * Detected change in %r, reloading' % filename)
  94. sys.exit(3)
  95. class StatReloaderLoop(ReloaderLoop):
  96. name = 'stat'
  97. def run(self):
  98. mtimes = {}
  99. while 1:
  100. for filename in chain(_iter_module_files(), self.extra_files):
  101. try:
  102. mtime = os.stat(filename).st_mtime
  103. except OSError:
  104. continue
  105. old_time = mtimes.get(filename)
  106. if old_time is None:
  107. mtimes[filename] = mtime
  108. continue
  109. elif mtime > old_time:
  110. self.trigger_reload(filename)
  111. self._sleep(self.interval)
  112. class WatchdogReloaderLoop(ReloaderLoop):
  113. def __init__(self, *args, **kwargs):
  114. ReloaderLoop.__init__(self, *args, **kwargs)
  115. from watchdog.observers import Observer
  116. from watchdog.events import FileSystemEventHandler
  117. self.observable_paths = set()
  118. def _check_modification(filename):
  119. if filename in self.extra_files:
  120. self.trigger_reload(filename)
  121. dirname = os.path.dirname(filename)
  122. if dirname.startswith(tuple(self.observable_paths)):
  123. if filename.endswith(('.pyc', '.pyo')):
  124. self.trigger_reload(filename[:-1])
  125. elif filename.endswith('.py'):
  126. self.trigger_reload(filename)
  127. class _CustomHandler(FileSystemEventHandler):
  128. def on_created(self, event):
  129. _check_modification(event.src_path)
  130. def on_modified(self, event):
  131. _check_modification(event.src_path)
  132. reloader_name = Observer.__name__.lower()
  133. if reloader_name.endswith('observer'):
  134. reloader_name = reloader_name[:-8]
  135. reloader_name += ' reloader'
  136. self.name = reloader_name
  137. self.observer_class = Observer
  138. self.event_handler = _CustomHandler()
  139. self.should_reload = False
  140. def trigger_reload(self, filename):
  141. # This is called inside an event handler, which means we can't throw
  142. # SystemExit here. https://github.com/gorakhargosh/watchdog/issues/294
  143. self.should_reload = True
  144. ReloaderLoop.trigger_reload(self, filename)
  145. def run(self):
  146. watches = {}
  147. observer = self.observer_class()
  148. observer.start()
  149. while not self.should_reload:
  150. to_delete = set(watches)
  151. paths = _find_observable_paths(self.extra_files)
  152. for path in paths:
  153. if path not in watches:
  154. try:
  155. watches[path] = observer.schedule(
  156. self.event_handler, path, recursive=True)
  157. except OSError:
  158. # "Path is not a directory". We could filter out
  159. # those paths beforehand, but that would cause
  160. # additional stat calls.
  161. watches[path] = None
  162. to_delete.discard(path)
  163. for path in to_delete:
  164. watch = watches.pop(path, None)
  165. if watch is not None:
  166. observer.unschedule(watch)
  167. self.observable_paths = paths
  168. self._sleep(self.interval)
  169. sys.exit(3)
  170. reloader_loops = {
  171. 'stat': StatReloaderLoop,
  172. 'watchdog': WatchdogReloaderLoop,
  173. }
  174. try:
  175. __import__('watchdog.observers')
  176. except ImportError:
  177. reloader_loops['auto'] = reloader_loops['stat']
  178. else:
  179. reloader_loops['auto'] = reloader_loops['watchdog']
  180. def run_with_reloader(main_func, extra_files=None, interval=1,
  181. reloader_type='auto'):
  182. """Run the given function in an independent python interpreter."""
  183. import signal
  184. reloader = reloader_loops[reloader_type](extra_files, interval)
  185. signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
  186. try:
  187. if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
  188. t = threading.Thread(target=main_func, args=())
  189. t.setDaemon(True)
  190. t.start()
  191. reloader.run()
  192. else:
  193. sys.exit(reloader.restart_with_reloader())
  194. except KeyboardInterrupt:
  195. pass