multithreading - Why threads will quit occasionally in my python program -

the program create monitor thread function: thread_monitor. monitor thread create hundreds of treads fucntion: valid_proxy , thread counte number of active threads every second.

but after run several hours, found num of active threds shrinking, 500 472 example. longer run time, more number of reduction.

i don't know what's wrong in function valid_proxy result abnormal thread exists. me point potential errors?

all code placed in: https://github.com/iaston/proxy_checker . here code fragments.

import threading import time import requests import requests.exceptions import requests.adapters import datetime import queue import re  def update_gui():     while true:         time.sleep(1)         num_of_threads = 0         each_thread in threading.enumerate():             if each_thread.name.find("verify_proxy_") == 0:                 num_of_threads += 1                          print("\nnumber of running threads %d.\n" % num_of_threads)   def get_a_proxy():     global g_b_stop, g_all_statu, g_proxy_queue, lock_get_proxy     if g_b_stop:           return ""     proxy_now = ""     lock_get_proxy.acquire()     try:         while proxy_now == "":             proxy_now = re.sub("[^\d:\.].+", "", str(g_proxy_queue.get(block=false)))     except exception:         proxy_now = ""     lock_get_proxy.release()     return proxy_now   def valid_proxy(check_site_info, success_try):     global lock_valided_list, g_tree_proxies, g_all_statu, proxies_valided_list     i_error_limit = success_try[1] - success_try[0]     if i_error_limit < 0:         i_error_limit = 0     i_error_now = 0     proxy_now = get_a_proxy()     while proxy_now != "":         test_num = 0         each_check_site in check_site_info:                i_error_now = 0             proxy_speed_recorder = {each_check_site: []}             icounter in range(0, success_try[1]):                   test_num += 1                 if (datetime.datetime.now() - g_gui_last_update_time).seconds > g_gui_update_interval:                     redraw_gui_event_finished.clear()                 redraw_gui_event_finished.wait()                   try:                     read_timeout = int(check_site_info[each_check_site]['timeout'])                     connect_timeout = read_timeout / 2                     if connect_timeout < 2:                         connect_timeout = 2                     start_test_time = time.time()                     req_headers = {                         "user-agent": "mozilla/5.0 (windows nt 6.3; wow64; rv:38.0) gecko/20100101 firefox/38.0",                         "referer": check_site_info[each_check_site]['url']}                     req_result = requests.get(check_site_info[each_check_site]['url'],                                               timeout=(connect_timeout, read_timeout),                                               proxies={'http': proxy_now, 'https': proxy_now}, headers=req_headers)                     used_time_seconds = (time.time() - start_test_time) * 1000                      html_result = req_result.text                 except exception e:                     used_time_seconds = -1                     html_result = ""                     print("\nerror in proxy %s:\n%r" % (proxy_now, e))                 if html_result.find(check_site_info[each_check_site]['keyword']) < 0:                     i_error_now += 1                     if i_error_now > i_error_limit:                         print(('\ninvalided proxy: ' + proxy_now))                         break                   else:                     proxy_speed_recorder[each_check_site].append((icounter, used_time_seconds))                      if icounter + 1 - i_error_now >= success_try[0]:                         break             if i_error_now > i_error_limit:                 break           print("proxy: " + proxy_now + " test number:" + str(test_num))         if i_error_now <= i_error_limit:              all_used_time = 0             all_test_time = 0             each_check_site in proxy_speed_recorder:                 each_test in proxy_speed_recorder[each_check_site]:                     all_used_time += each_test[1]                     all_test_time += 1             avarge_time = round(all_used_time / all_test_time) if all_test_time != 0 else 0             lock_valided_list.acquire()             proxies_valided_list.append((proxy_now, avarge_time))             try:                 g_all_statu["text_proxy_valid_append"] += proxy_now + "&" + str(avarge_time) + "\n"                   g_tree_proxies.add_data_treeview([(proxy_now, avarge_time)], skip_datebase=true)               except exception e2:                 print("""g_all_statu["text_proxy_valid_append"] wrong:\n""" + repr(e2))             lock_valided_list.release()         time.sleep(1)         proxy_now = get_a_proxy()     print("finish, thread exit")   def thread_monitor(check_site_info, success_try, th_num):     each_proxy in proxies_unvalided_list:         g_proxy_queue.put(each_proxy)      th_gui = threading.thread(target=update_gui)     th_gui.setdaemon(true)     th_gui.start()      ths_verify = []     icounter in range(0, th_num):         t = threading.thread(target=valid_proxy, args=(check_site_info, success_try))         t.setdaemon(true)         t.setname("verify_proxy_" + str(icounter))         t.start()         ths_verify.append(t)      icounter in range(0, th_num):         ths_verify[icounter].join()   thread_monitor(check_site_info, success_try, 500) 

i can't troubleshoot function valid_proxy since references code didn't provide , i'm not clairvoyant. however, can tell if exception occurs anywhere in function cause thread exit. if this:

def forever_valid_proxy(*x,**y):     try:          valid_proxy(*x,**y)     except exception:          pass 

and replace line:

t = threading.thread(target=valid_proxy, args=(check_site_info, success_try)) 

with:

t = threading.thread(target=forever_valid_proxy, args=(check_site_info, success_try)) 

the thread count never decrease. whether program want or need don't know, never see thread count decreasing.