m

douyin_video_scraper_web.py

@@ -40,6 +40,11 @@ if __name__ == '__main__':
         
         rds = RdsUserVideoList()
 
+        ck = rds.get_cookie()
+        if ck is None:
+            ck = webdy.get_ck(proxy=proxies)
+            rds.set_cookie(ck)
+
         for index in range(1000):
             try:
                 # ukey = 'BrandLiveData.DouyinUserVideo'
@@ -47,7 +52,7 @@ if __name__ == '__main__':
                 users = rds.get_request_param()
                 
                 if users is None:
-                    time.sleep(10)
+                    time.sleep(1)
                     continue
                 user = json.loads(users)
                 user_id = user.get('uid')
@@ -60,7 +65,7 @@ if __name__ == '__main__':
                 })
                 
                 users = json.dumps(user)
-                videos = webdy.get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=20,proxy=proxies)
+                videos = webdy.get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=20,proxy=proxies,cookie=ck)
                 # skey = 'BrandLiveData.DouyinUserVideoScore'
                 # data_score = DbRedis.connect().rpop(skey)
                 
@@ -147,3 +152,4 @@ if __name__ == '__main__':
                     + "\n"
                 )
                 break
+

rds_model/rds_user_video_list.py

@@ -33,4 +33,13 @@ class RdsUserVideoList:
     # 获取现有记录
     def get_score(self):
         key = 'BrandLiveData.DouyinUserVideoScore'
-        return self.redis.get(key)
+        return self.redis.get(key)
+    
+    def get_cookie(self):
+        key = 'BrandLiveData.DouyinUserVideoWebCookie'
+        return self.redis.get(key)
+
+    def set_cookie(self, data):
+        key = 'BrandLiveData.DouyinUserVideoWebCookie'
+        self.redis.set(key, data)
+        self.redis.expire(key, 7200)

web_dy.py

@@ -18,7 +18,7 @@ class WebDouYin:
         
         # print('_signature',d)
         return d
-    def get_ck(self):
+    def get_ck(self, proxy=None):
         requests.packages.urllib3.disable_warnings()
         headers = {
             'authority': 'www.douyin.com',
@@ -31,7 +31,7 @@ class WebDouYin:
             'upgrade-insecure-requests': '1',
             'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
         }
-        res = requests.get('https://www.douyin.com/', headers=headers,verify=False)
+        res = requests.get('https://www.douyin.com/', headers=headers,verify=False, proxies=proxy)
         cookie_dict=res.cookies.get_dict()
         ck=''
         if cookie_dict:
@@ -65,9 +65,12 @@ class WebDouYin:
             json_data = "请求失败" + str(e)
             print(json_data)
         return json_data
-    def get_user_videos(self,sec_user_id,max_cursor=0,count=10,proxy=None):
+    def get_user_videos(self,sec_user_id,max_cursor=0,count=10,proxy=None,cookie=None):
         
         ua,ck=self.get_ua_ck('get_user_videos')
+
+        if cookie:
+            ck = cookie
         
         url='https://www.douyin.com/aweme/v1/web/aweme/post/?'
         param={