m

douyin_video_scraper_web.py

@@ -11,7 +11,7 @@ from rds_model.rds_user_video_list import RdsUserVideoList
 if __name__ == '__main__':
     import warnings
     start_time = time.time()
-    print('1')
+    
     warnings.filterwarnings("ignore")
     while True:
 
@@ -31,20 +31,19 @@ if __name__ == '__main__':
             "http": "http://" + proxy,
             "https": "http://" + proxy
         }
-        print('2')
+        
         errorn = 0
         suc_f = 0
         webdy=WebDouYin()
-        print('3')
+        
         rds = RdsUserVideoList()
-        print('4')
 
         for index in range(1000):
             try:
                 # ukey = 'BrandLiveData.DouyinUserVideo'
                 # users = DbRedis.connect().rpop(ukey)
                 users = rds.get_request_param()
-                print('5')
+                
                 if users is None:
                     time.sleep(10)
                     continue
@@ -61,19 +60,18 @@ if __name__ == '__main__':
                 videos = webdy.get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=20,proxy=proxies)
                 # skey = 'BrandLiveData.DouyinUserVideoScore'
                 # data_score = DbRedis.connect().rpop(skey)
-                print('6')
+                
                 data_score = rds.get_score()
                 if data_score is None:
-                    data_score = {"total" : 1, "success" : 0, "fail" : 0}
+                    data_score = '1@@@0@@@0'
                 else:
-                    data_score = data_score.loads(data_score)
-                    total = data_score.get('total')
-                    data_score.update({
-                        'total' : total + 1
-                        })
+                    data_score = data_score.split('@@@')
+                    total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
+                    total = total + 1
+                    data_score = total + '@@@' + success + '@@@' + fail
                 rds.record_score(data_score)
                 # DbRedis.connect().lpush(skey, data_score)
-                print('7')
+                
                 awemes = videos.get('aweme_list')
                 stime = time.strftime("%H:%M:%S", time.localtime())
                 if awemes:
@@ -88,13 +86,12 @@ if __name__ == '__main__':
                     # data_score = DbRedis.connect().rpop(skey)
                     data_score = rds.get_score()
                     if data_score is None:
-                        data_score = {"total" : 1, "success" : 1, "fail" : 0}
+                        data_score = '1@@@1@@@0'
                     else:
-                        data_score = data_score.loads(data_score)
-                        success = data_score.get('success')
-                        data_score.update({
-                            'success' : success + 1
-                            })
+                        data_score = data_score.split('@@@')
+                        total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
+                        success = success + 1
+                        data_score = total + '@@@' + success + '@@@' + fail
                     # DbRedis.connect().lpush(skey, data_score)
                     rds.record_score(data_score)
 
@@ -103,13 +100,12 @@ if __name__ == '__main__':
                     # data_score = DbRedis.connect().rpop(skey)
                     data_score = rds.get_score()
                     if data_score is None:
-                        data_score = {"total" : 1, "success" : 0, "fail" : 1}
+                        data_score = '1@@@0@@@1'
                     else:
-                        data_score = data_score.loads(data_score)
-                        fail = data_score.get('fail')
-                        data_score.update({
-                            'fail' : fail + 1
-                            })
+                        data_score = data_score.split('@@@')
+                        total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
+                        fail = fail + 1
+                        data_score = total + '@@@' + success + '@@@' + fail
                     # DbRedis.connect().lpush(skey, data_score) 
                     rds.record_score(data_score)
                     # PrintLog.print(stime+" 失败"+str(index)+' '+ str(sec_user_id))
@@ -126,13 +122,12 @@ if __name__ == '__main__':
                 # data_score = DbRedis.connect().rpop(skey)
                 data_score = rds.get_score()
                 if data_score is None:
-                    data_score = {"total" : 1, "success" : 0, "fail" : 1}
+                    data_score = '1@@@0@@@1'
                 else:
-                    data_score = data_score.loads(data_score)
-                    fail = data_score.get('fail')
-                    data_score.update({
-                        'fail' : fail + 1
-                        })
+                    data_score = data_score.split('@@@')
+                    total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
+                    fail = fail + 1
+                    data_score = total + '@@@' + success + '@@@' + fail
                 # DbRedis.connect().lpush(skey, data_score) 
                 rds.record_score(data_score)
                 print(

web_dy.py

@@ -8,15 +8,15 @@ class WebDouYin:
         self.proxies = {
         }
     def get_signature(self,url=None,method='_signature'):
-        print('5-3-1')
+        
         with open('signature.js', 'r', encoding='utf-8') as f:
             b = f.read()
-        print('5-3-2')
+        
         c = execjs.compile(b)
-        print('5-3-3')
+        
         # url=url.replace('%28','(').replace('%29',')').replace('%2C',',')
         d = c.call(method, url.replace('\n',''))
-        print('5-3-4')
+        
         # print('_signature',d)
         return d
     def get_ck(self):
@@ -63,9 +63,9 @@ class WebDouYin:
             print(json_data)
         return json_data
     def get_user_videos(self,sec_user_id,max_cursor=0,count=10,proxy=None):
-        print('5-1')
+        
         ua,ck=self.get_ua_ck('get_user_videos')
-        print('5-2')
+        
         url='https://www.douyin.com/aweme/v1/web/aweme/post/?'
         param={
             'device_platform': 'webapp',
@@ -88,10 +88,10 @@ class WebDouYin:
         }
 
         url = url + parse.urlencode(param)
-        print('5-3')
+        
         _signature = self.get_signature(url)
         url+='&_signature='+quote(_signature)
-        print('5-4')
+        
         headers = {
             "authority": "www.douyin.com",
             "method": "GET",
@@ -107,7 +107,7 @@ class WebDouYin:
         }
         if ck:headers['cookie']=ck
         json_data=self.response(url=url,headers=headers,proxy=proxy)
-        print('5-5')
+        
         return json_data
         # print(json_data)
     def get_user_info(self,sec_user_id,proxy=None):