python multiprocessing pool terminate
I found solution: stop pool in separate thread, like this:
def close_pool():
global pool
pool.close()
pool.terminate()
pool.join()
def term(*args,**kwargs):
sys.stderr.write('\nStopping...')
# httpd.shutdown()
stophttp = threading.Thread(target=httpd.shutdown)
stophttp.start()
stoppool=threading.Thread(target=close_pool)
stoppool.daemon=True
stoppool.start()
signal.signal(signal.SIGTERM, term)
signal.signal(signal.SIGINT, term)
signal.signal(signal.SIGQUIT, term)
Works fine and always i tested.
signal.SIGINT
Interrupt from keyboard (CTRL + C). Default action is to raise KeyboardInterrupt.
signal.SIGKILL
Kill signal. It cannot be caught, blocked, or ignored.
signal.SIGTERM
Termination signal.
signal.SIGQUIT
Quit with core dump.
If you're still experiencing this issue, you could try simulating a Pool
with daemonic processes (assuming you are starting the pool/processes from a non-daemonic process). I doubt this is the best solution since it seems like your Pool
processes should be exiting, but this is all I could come up with. I don't know what your callback does so I'm not sure where to put it in my example below.
I also suggest trying to create your Pool
in __main__
due to my experience (and the docs) with weirdness occurring when processes are spawned globally. This is especially true if you're on Windows: http://docs.python.org/2/library/multiprocessing.html#windows
from multiprocessing import Process, JoinableQueue
# the function for each process in our pool
def pool_func(q):
while True:
allRenderArg, otherArg = q.get() # blocks until the queue has an item
try:
render(allRenderArg, otherArg)
finally: q.task_done()
# best practice to go through main for multiprocessing
if __name__=='__main__':
# create the pool
pool_size = 2
pool = []
q = JoinableQueue()
for x in range(pool_size):
pool.append(Process(target=pool_func, args=(q,)))
# start the pool, making it "daemonic" (the pool should exit when this proc exits)
for p in pool:
p.daemon = True
p.start()
# submit jobs to the queue
for i in range(totalInstances):
q.put((allRenderArgs[i], args[2]))
# wait for all tasks to complete, then exit
q.join()