Merge pull request jhao104#131 from highroom/branch1

modify proxy valid use queue

Author: jhao104

Schedule/ProxyCheck.py

@@ -13,7 +13,7 @@
 __author__ = 'J_hao'
 
 import sys
-from time import sleep
+import threading
 from threading import Thread
 
 sys.path.append('../')
@@ -26,32 +26,36 @@
 
 
 class ProxyCheck(ProxyManager, Thread):
-    def __init__(self):
+    def __init__(self, queue, item_dict):
         ProxyManager.__init__(self)
         Thread.__init__(self)
         self.log = LogHandler('proxy_check')
+        self.queue = queue
+        self.item_dict = item_dict
 
     def run(self):
         self.db.changeTable(self.useful_proxy_queue)
-        while True:
-            for proxy, count in self.db.getAll().items():
-                if validUsefulProxy(proxy):
-                    # 验证通过计数器减1
-                    if count and int(count) > 0:
-                        self.db.put(proxy, num=int(count) - 1)
-                    else:
-                        pass
-                    self.log.info('ProxyCheck: {} validation pass'.format(proxy))
+        while self.queue.qsize():
+            proxy = self.queue.get()
+            count = self.item_dict[proxy]
+            if validUsefulProxy(proxy):
+                # 验证通过计数器减1
+                if count and int(count) > 0:
+                    self.db.put(proxy, num=int(count) - 1)
                 else:
-                    self.log.info('ProxyCheck: {} validation fail'.format(proxy))
-                    if count and int(count) > FAIL_COUNT:
-                        self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
-                        self.db.delete(proxy)
-                    else:
-                        self.db.put(proxy, num=int(count) + 1)
-            sleep(60 * 5)
+                    pass
+                print('ProxyCheck: {} validation pass'.format(proxy))
+            else:
+                print('ProxyCheck: {} validation fail'.format(proxy))
+                if count and int(count) > FAIL_COUNT:
+                    print('ProxyCheck: {} fail too many, delete!'.format(proxy))
+                    self.db.delete(proxy)
+                else:
+                    self.db.put(proxy, num=int(count) + 1)
+            self.queue.task_done()
 
 
 if __name__ == '__main__':
-    p = ProxyCheck()
-    p.run()
+    # p = ProxyCheck()
+    # p.run()
+    pass

Schedule/ProxyValidSchedule.py

@@ -17,21 +17,25 @@
 sys.path.append('../')
 
 from Schedule.ProxyCheck import ProxyCheck
+from Manager.ProxyManager import ProxyManager
+from queue import Queue
+import time
 
 
-class ProxyValidSchedule(object):
+class ProxyValidSchedule(ProxyManager, object):
     def __init__(self):
-        pass
+        ProxyManager.__init__(self)
+        self.queue = Queue()
 
-    def __validProxy(self, threads=5):
+    def __validProxy(self, threads=10):
         """
         验证useful_proxy代理
         :param threads: 线程数
         :return:
         """
         thread_list = list()
         for index in range(threads):
-            thread_list.append(ProxyCheck())
+            thread_list.append(ProxyCheck(self.queue, self.item_dict))
 
         for thread in thread_list:
             thread.daemon = True
@@ -41,7 +45,20 @@ def __validProxy(self, threads=5):
             thread.join()
 
     def main(self):
-        self.__validProxy()
+        self.put_queue()
+        while True:
+            if self.queue.qsize():
+                self.__validProxy()
+            else:
+                print('Time sleep 5 minutes.')
+                time.sleep(60 * 1)
+                self.put_queue()
+
+    def put_queue(self):
+        self.db.changeTable(self.useful_proxy_queue)
+        self.item_dict = self.db.getAll()
+        for item in self.item_dict:
+            self.queue.put(item)
 
 
 def run():
@@ -51,4 +68,4 @@ def run():
 
 if __name__ == '__main__':
     p = ProxyValidSchedule()
-    p.main()
+    p.main()