Merge branch 'master' of http://101.200.220.49:8001/houxiaohua/shop_live_scraper

douyin_hourly_ranklist_commerce_scraper.py

@@ -81,7 +81,7 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
     query = {
         "style" : "3",
         "hour_info" : "0",
-        "room_id" : room_id,
+        "room_id" : str(room_id),
         "rank_type" : "31",
         "sec_anchor_id" : sec_anchor_id,
         "webcast_sdk_version" : "1710",
@@ -95,7 +95,7 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
         "dpi" : "480",
         "app_name" : "aweme",
         "version_name" : "13.0.0",
-        "ts" : ts,
+        "ts" : str(ts),
         "cpu_support64" : "true",
         "storage_type" : "0",
         "app_type" : "normal",
@@ -106,12 +106,12 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
         "channel" : "tengxun_new",
         "_rticket" : rticket,
         "device_platform" : "android",
-        "iid" : iid,
+        "iid" : str(iid),
         "version_code" : "130000",
         "mac_address" : mc,
         "cdid" : "81542dc6-2aca-4ff6-ac58-d94179e9d3e6",
-        "openudid" : openudid,
-        "device_id" : device_id,
+        "openudid" : str(openudid),
+        "device_id" : str(device_id),
         "resolution" : "1080*1800",
         "os_version" : "6.0",
         "language" : "zh",
@@ -126,8 +126,7 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
 
     xGorgon = X_Gorgon(query_params, body)
 
-    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
-        random.randint(1, 10))
+    userAgent = f'com.ss.android.ugc.aweme/1300000 (Linux; U; Android 5.1.1; zh_CN; {device_type}; Build/LMY47V; Cronet/58.0.2991.0)'
 
     headers = {
         'Host': domain,
@@ -141,13 +140,14 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
         'sdk-version' : '2',
         'x-ss-dp' : '1128',
         'x-tt-trace-id' : trace_id,
+        'cookie' : cookie
     }
 
     retry = 0
     response_json = None
 
     while True:
-        if retry > 50:
+        if retry > 0:
             break
 
         retry += 1
@@ -231,42 +231,52 @@ def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
 def scrape(room_id,sec_anchor_id,anchor_id):
     rds_list = RdsDouyinHourlyRankList()
 
-    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
-    rdid = DbRedis.connect().rpop(key)
-    if rdid:
-        result = rdid.split('@@@')
-    else:
-        result = []
-        return None
+    start_time = int(time.time())
 
-    DbRedis.connect().lpush(key, rdid)
+    while True:
 
-    try:
-        # 带货小时榜
-        commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
-        
-        if commerce_response_json is None:
+        current_time = int(time.time())
+        if current_time - start_time >= 9*60:
             print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 带货小时榜数据获取失败！响应数据为空！' 
-            )
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 带货小时榜数据获取失败！响应数据为空！' 
+                )
             sys.exit(0)
+
+        key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+        rdid = DbRedis.connect().rpop(key)
+        if rdid:
+            result = rdid.split('@@@')
         else:
-            data = json.dumps({
-                "data": commerce_response_json.get('data'),
-                "extra": {}
-            })
-            rds_list.push_commerce_data_list(data)
-
-    except Exception as e:
-        print(
-            time.strftime("%H:%M:%S", time.localtime()) 
-            + ' ' 
-            + '数据异常：' 
-            + str(e)
-        )
-
-    sys.exit(0)
+            result = []
+            return None
+
+        DbRedis.connect().lpush(key, rdid)
+
+        try:
+            # 带货小时榜
+            commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
+            
+            if commerce_response_json is None:
+                print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 带货小时榜数据获取失败！响应数据为空！' 
+                )
+            else:
+                data = json.dumps({
+                    "data": commerce_response_json.get('data'),
+                    "extra": {}
+                })
+                rds_list.push_commerce_data_list(data)
+                sys.exit(0)
+
+        except Exception as e:
+            print(
+                time.strftime("%H:%M:%S", time.localtime()) 
+                + ' ' 
+                + '数据异常：' 
+                + str(e)
+            )
 
 if __name__ == "__main__":
     print("主方法开始执行")

douyin_live_lottery_scraper.py

@@ -78,13 +78,13 @@ def get_live_lottery_data(room_id, result):
     device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11]
     
     query = {
-        "room_id" : room_id,
+        "room_id" : str(room_id),
         "webcast_sdk_version" : "1690",
         "webcast_language" : "zh",
         "webcast_locale" :"zh_CN",
         "webcast_gps_access" : "1",
         "os_api" : "23",
-        "device_type" : "SM-G9200",
+        "device_type" : device_type,
         "ssmix" : "a",
         "manifest_version_code" : "120801",
         "dpi" : "640",
@@ -102,17 +102,17 @@ def get_live_lottery_data(room_id, result):
         "_rticket" : rticket,
         # "_rticket" : "1629688012123",
         "device_platform" : "android",
-        "iid" : iid,
+        "iid" : str(iid),
         "version_code" : "120800",
         "mac_address" : mc,
         #"mac_address" : "FC%3ADB%3AB3%3A56%3ABD%3AFD",
         "cdid" : "6c96979e-c729-419c-9516-3a85a7338d0c",
-        "openudid" : openudid,
-        "device_id" : device_id,
+        "openudid" : str(openudid),
+        "device_id" : str(device_id),
         "resolution" : "1440*2560",
         "os_version" : "6.0.1",
         "language" : "zh",
-        "device_brand":"samsung",
+        "device_brand":device_brand,
         "aid" : "1128"
     }
 
@@ -123,8 +123,7 @@ def get_live_lottery_data(room_id, result):
 
     xGorgon = X_Gorgon(query_params, body)
 
-    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
-        random.randint(1, 10))
+    userAgent = userAgent = f'com.ss.android.ugc.aweme/1208000 (Linux; U; Android 5.1.1; zh_CN; {device_type}; Build/LMY47V; Cronet/58.0.2991.0)'
 
     headers = {
         'Host': domain,