test

douyin_hourly_ranklist_commerce_scraper.py

@@ -231,42 +231,52 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
 def scrape(room_id,sec_anchor_id,anchor_id):
     rds_list = RdsDouyinHourlyRankList()
 
-    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
-    rdid = DbRedis.connect().rpop(key)
-    if rdid:
-        result = rdid.split('@@@')
-    else:
-        result = []
-        return None
+    start_time = int(time.time())
 
-    DbRedis.connect().lpush(key, rdid)
+    while True:
 
-    try:
-        # 带货小时榜
-        commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
-        
-        if commerce_response_json is None:
+        current_time = int(time.time())
+        if current_time - start_time >= 9*60:
             print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 带货小时榜数据获取失败！响应数据为空！' 
-            )
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 带货小时榜数据获取失败！响应数据为空！' 
+                )
             sys.exit(0)
+
+        key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+        rdid = DbRedis.connect().rpop(key)
+        if rdid:
+            result = rdid.split('@@@')
         else:
-            data = json.dumps({
-                "data": commerce_response_json.get('data'),
-                "extra": {}
-            })
-            rds_list.push_commerce_data_list(data)
-
-    except Exception as e:
-        print(
-            time.strftime("%H:%M:%S", time.localtime()) 
-            + ' ' 
-            + '数据异常：' 
-            + str(e)
-        )
-
-    sys.exit(0)
+            result = []
+            return None
+
+        DbRedis.connect().lpush(key, rdid)
+
+        try:
+            # 带货小时榜
+            commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
+            
+            if commerce_response_json is None:
+                print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 带货小时榜数据获取失败！响应数据为空！' 
+                )
+            else:
+                data = json.dumps({
+                    "data": commerce_response_json.get('data'),
+                    "extra": {}
+                })
+                rds_list.push_commerce_data_list(data)
+                sys.exit(0)
+
+        except Exception as e:
+            print(
+                time.strftime("%H:%M:%S", time.localtime()) 
+                + ' ' 
+                + '数据异常：' 
+                + str(e)
+            )
 
 if __name__ == "__main__":
     print("主方法开始执行")