multithreading - Why threads will quit occasionally in my python program -


the program create monitor thread function: thread_monitor. monitor thread create hundreds of treads fucntion: valid_proxy , thread counte number of active threads every second.

but after run several hours, found num of active threds shrinking, 500 472 example. longer run time, more number of reduction.

i don't know what's wrong in function valid_proxy result abnormal thread exists. me point potential errors?

all code placed in: https://github.com/iaston/proxy_checker . here code fragments.

import threading import time import requests import requests.exceptions import requests.adapters import datetime import queue import re  def update_gui():     while true:         time.sleep(1)         num_of_threads = 0         each_thread in threading.enumerate():             if each_thread.name.find("verify_proxy_") == 0:                 num_of_threads += 1                          print("\nnumber of running threads %d.\n" % num_of_threads)   def get_a_proxy():     global g_b_stop, g_all_statu, g_proxy_queue, lock_get_proxy     if g_b_stop:           return ""     proxy_now = ""     lock_get_proxy.acquire()     try:         while proxy_now == "":             proxy_now = re.sub("[^\d:\.].+", "", str(g_proxy_queue.get(block=false)))     except exception:         proxy_now = ""     lock_get_proxy.release()     return proxy_now   def valid_proxy(check_site_info, success_try):     global lock_valided_list, g_tree_proxies, g_all_statu, proxies_valided_list     i_error_limit = success_try[1] - success_try[0]     if i_error_limit < 0:         i_error_limit = 0     i_error_now = 0     proxy_now = get_a_proxy()     while proxy_now != "":         test_num = 0         each_check_site in check_site_info:                i_error_now = 0             proxy_speed_recorder = {each_check_site: []}             icounter in range(0, success_try[1]):                   test_num += 1                 if (datetime.datetime.now() - g_gui_last_update_time).seconds > g_gui_update_interval:                     redraw_gui_event_finished.clear()                 redraw_gui_event_finished.wait()                   try:                     read_timeout = int(check_site_info[each_check_site]['timeout'])                     connect_timeout = read_timeout / 2                     if connect_timeout < 2:                         connect_timeout = 2                     start_test_time = time.time()                     req_headers = {                         "user-agent": "mozilla/5.0 (windows nt 6.3; wow64; rv:38.0) gecko/20100101 firefox/38.0",                         "referer": check_site_info[each_check_site]['url']}                     req_result = requests.get(check_site_info[each_check_site]['url'],                                               timeout=(connect_timeout, read_timeout),                                               proxies={'http': proxy_now, 'https': proxy_now}, headers=req_headers)                     used_time_seconds = (time.time() - start_test_time) * 1000                      html_result = req_result.text                 except exception e:                     used_time_seconds = -1                     html_result = ""                     print("\nerror in proxy %s:\n%r" % (proxy_now, e))                 if html_result.find(check_site_info[each_check_site]['keyword']) < 0:                     i_error_now += 1                     if i_error_now > i_error_limit:                         print(('\ninvalided proxy: ' + proxy_now))                         break                   else:                     proxy_speed_recorder[each_check_site].append((icounter, used_time_seconds))                      if icounter + 1 - i_error_now >= success_try[0]:                         break             if i_error_now > i_error_limit:                 break           print("proxy: " + proxy_now + " test number:" + str(test_num))         if i_error_now <= i_error_limit:              all_used_time = 0             all_test_time = 0             each_check_site in proxy_speed_recorder:                 each_test in proxy_speed_recorder[each_check_site]:                     all_used_time += each_test[1]                     all_test_time += 1             avarge_time = round(all_used_time / all_test_time) if all_test_time != 0 else 0             lock_valided_list.acquire()             proxies_valided_list.append((proxy_now, avarge_time))             try:                 g_all_statu["text_proxy_valid_append"] += proxy_now + "&" + str(avarge_time) + "\n"                   g_tree_proxies.add_data_treeview([(proxy_now, avarge_time)], skip_datebase=true)               except exception e2:                 print("""g_all_statu["text_proxy_valid_append"] wrong:\n""" + repr(e2))             lock_valided_list.release()         time.sleep(1)         proxy_now = get_a_proxy()     print("finish, thread exit")   def thread_monitor(check_site_info, success_try, th_num):     each_proxy in proxies_unvalided_list:         g_proxy_queue.put(each_proxy)      th_gui = threading.thread(target=update_gui)     th_gui.setdaemon(true)     th_gui.start()      ths_verify = []     icounter in range(0, th_num):         t = threading.thread(target=valid_proxy, args=(check_site_info, success_try))         t.setdaemon(true)         t.setname("verify_proxy_" + str(icounter))         t.start()         ths_verify.append(t)      icounter in range(0, th_num):         ths_verify[icounter].join()   thread_monitor(check_site_info, success_try, 500) 

i can't troubleshoot function valid_proxy since references code didn't provide , i'm not clairvoyant. however, can tell if exception occurs anywhere in function cause thread exit. if this:

def forever_valid_proxy(*x,**y):     try:          valid_proxy(*x,**y)     except exception:          pass 

and replace line:

t = threading.thread(target=valid_proxy, args=(check_site_info, success_try)) 

with:

t = threading.thread(target=forever_valid_proxy, args=(check_site_info, success_try)) 

the thread count never decrease. whether program want or need don't know, never see thread count decreasing.


Comments

Popular posts from this blog

Rendering JButton to get the JCheckBox behavior in a JTable by using images does not update my table -

javascript - Using jquery append to add option values into a select element not working -

Android soft keyboard reverts to default keyboard on orientation change -