Merge branch 'master' of http://101.200.220.49:8001/houxiaohua/shop_live_scraper

live_promotion_detail_v1_sale_scraper.py

@@ -33,30 +33,6 @@ def scrape(request_data, heat):
         sp_ns = exec_time - int(time.time())
         time.sleep(sp_ns)
 
-    if heat < 2:
-        goodsDb = MysqlDyLiveCommodity()
-        pix = str(uid)[-1]
-        goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
-
-        dataid,sold_out_at,revise_price,created_at = goodsInfo
-
-        print(goodsInfo)
-
-        if revise_price is not None:
-            sys.exit(0)
-
-
-        if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
-            add_t = 300
-            news_time = int(time.time()) + add_t
-            next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
-            rds_list.push_middle_list(next_rds)
-        else :
-            add_t = 1200
-            news_time = int(time.time()) + add_t
-            next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
-            rds_list.push_last_list(next_rds)
-
 
     PrintLog.print(
         time.strftime("%H:%M:%S", time.localtime()) + ' '
@@ -68,15 +44,49 @@ def scrape(request_data, heat):
 
         commodity_detail = LiveCommodityDetailV1.get_data(product_id)
 
-        if (commodity_detail is None) or (commodity_detail == ''):
+        if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False :
             rds_list.record_v1_score(0)
             PrintLog.print(
                 time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 数据为空 ' + promotion_id
             )
-            if random.randint(0, 9) > 0:
-                rds_list.add_request_params(request_data, heat)
 
+            if heat==0:
+                rds_list.add_request_params(request_data, heat)
+                print('首次失败重新插入 ' + request_data)
+                sys.exit(0)
+
+            if heat>0:
+                if random.randint(0, 9) > 0:
+                    rds_list.add_request_params(request_data, heat)
+                    print(str(heat) + '非首次失败重新插入 ' + request_data)
+                    sys.exit(0)
+
+        if heat < 2:
+            goodsDb = MysqlDyLiveCommodity()
+            pix = str(uid)[-1]
+            goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
+
+            dataid,sold_out_at,revise_price,created_at = goodsInfo
+
+            if revise_price is not None:
+                sys.exit(0)
+
+            if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
+                add_t = 300
+                news_time = int(time.time()) + add_t
+                next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
+                rds_list.push_middle_list(next_rds)
+                print('中间插入 ' + next_rds)
+            else :
+                add_t = 1200
+                news_time = int(time.time()) + add_t
+                next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
+                rds_list.push_last_list(next_rds)
+                print('末次插入 ' + next_rds)
+
+        if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False:
             sys.exit(0)
+            print('失败----- ')
 
         data = json.dumps({
             "data": commodity_detail,
@@ -91,6 +101,7 @@ def scrape(request_data, heat):
 
         rds_list.record_v1_score(1)
         rds_list.push_v1_data_list(data)
+        print('成功+++++ ')
     except Exception as e:
         rds_list.record_v1_score(0)
         rds_list.add_request_params(request_data, heat)

rds_model/rds_live_commodity_detail_request_list.py

@@ -25,7 +25,7 @@ class RdsLiveCommodityDetailRequestList:
     def add_request_params(self, request_data, heat):
         key = self.params_redis_key_dict.get(heat)
 
-        self.redis.rpush(key, request_data)
+        return self.redis.rpush(key, request_data)
 
     def record_v1_score(self, data):
         key = 'BrandLiveData.DouyinLiveGoodsForSaleListScore'