Browse Source

Merge branch 'master' of http://101.200.220.49:8001/houxiaohua/shop_live_scraper

chenzhiyuan 3 years ago
parent
commit
33c35aaa10
2 changed files with 3 additions and 4 deletions
  1. 2 3
      douyin_user_scraper_in.py
  2. 1 1
      douyin_video_scraper_web.py

+ 2 - 3
douyin_user_scraper_in.py

92
             print('thrend_' + threading.current_thread().name + ' finish')
92
             print('thrend_' + threading.current_thread().name + ' finish')
93
             break
93
             break
94
 
94
 
95
-        ullen = redisDyModel.llen(uKey)
96
-        if ullen == 0:
95
+        user_id = redisDyModel.rpop(uKey)
96
+        if not user_id:
97
             time.sleep(2)
97
             time.sleep(2)
98
             continue
98
             continue
99
 
99
 
100
         str_time = time.strftime("%H:%M:%S", time.localtime())
100
         str_time = time.strftime("%H:%M:%S", time.localtime())
101
-        user_id = redisDyModel.rpop(uKey)
102
 
101
 
103
         try:
102
         try:
104
             proxy = get_scraper_proxy(pKey)
103
             proxy = get_scraper_proxy(pKey)

+ 1 - 1
douyin_video_scraper_web.py

280
 
280
 
281
     except Exception as e:
281
     except Exception as e:
282
         set_score('fail')
282
         set_score('fail')
283
-        # rds.push_request_id(sec_user_id)
283
+        rds.push_request_id(sec_user_id)
284
         print(
284
         print(
285
             time.strftime("%H:%M:%S", time.localtime()) 
285
             time.strftime("%H:%M:%S", time.localtime()) 
286
             + ' ' 
286
             + ' '