live_promotion_detail_v1_sale_scraper.py

@@ -15,99 +15,105 @@ from log.print_log import PrintLog
 
 start_time = time.time()
 
-def scrape(request_data, heat):
-    rds_list = RdsLiveCommodityDetailRequestList()
-    request_split = request_data.split('@')
+def scrape(heat):
+    while True:
+        rds = RdsLiveCommodityDetailRequestList()
+        request_data = rds.get_request_params(heat)
+        if request_data is None:
+            time.sleep(0.1)
+            break
 
-    if len(request_split) < 5:
-        print('数据格式不对' + request_data)
-        sys.exit(0)
+        rds_list = RdsLiveCommodityDetailRequestList()
+        request_split = request_data.split('@')
 
-    promotion_id = request_split[0]
-    product_id = request_split[1]
-    uid = request_split[2]
-    room_id = request_split[3]
-    exec_time = int(request_split[4])
+        if len(request_split) < 5:
+            print('数据格式不对' + request_data)
+            sys.exit(0)
 
-    if int(time.time()) < exec_time:
-        sp_ns = exec_time - int(time.time())
-        time.sleep(sp_ns)
+        promotion_id = request_split[0]
+        product_id = request_split[1]
+        uid = request_split[2]
+        room_id = request_split[3]
+        exec_time = int(request_split[4])
 
+        if int(time.time()) < exec_time:
+            sp_ns = exec_time - int(time.time())
+            time.sleep(sp_ns)
 
-    PrintLog.print(
-        time.strftime("%H:%M:%S", time.localtime()) + ' '
-           + time.strftime("%H:%M:%S", time.localtime(exec_time)) + ' '
-           + str(heat) + ' ' + promotion_id
-    )
 
-    try:
+        PrintLog.print(
+            time.strftime("%H:%M:%S", time.localtime()) + ' '
+               + time.strftime("%H:%M:%S", time.localtime(exec_time)) + ' '
+               + str(heat) + ' ' + promotion_id
+        )
 
-        commodity_detail = LiveCommodityDetailV1.get_data(product_id)
+        try:
 
-        if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False :
-            rds_list.record_v1_score(0)
-            PrintLog.print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 数据为空 ' + promotion_id
-            )
+            commodity_detail = LiveCommodityDetailV1.get_data(product_id)
 
-            if heat==0:
-                if random.randint(0, 100) > 0:
-                    rds_list.add_request_params(request_data, heat)
-                    print('首次失败重新插入 ' + request_data)
-                    sys.exit(0)
+            if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False :
+                rds_list.record_v1_score(0)
+                PrintLog.print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 数据为空 ' + promotion_id
+                )
 
-            if heat>0:
-                if random.randint(0, 9) > 0:
-                    rds_list.add_request_params(request_data, heat)
-                    print(str(heat) + '非首次失败重新插入 ' + request_data)
-                    sys.exit(0)
+                if heat==0:
+                    if random.randint(0, 100) > 0:
+                        rds_list.add_request_params(request_data, heat)
+                        print('首次失败重新插入 ' + request_data)
+                        sys.exit(0)
 
-        if heat < 2:
-            goodsDb = MysqlDyLiveCommodity()
-            pix = str(uid)[-1]
-            goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
+                if heat>0:
+                    if random.randint(0, 9) > 0:
+                        rds_list.add_request_params(request_data, heat)
+                        print(str(heat) + '非首次失败重新插入 ' + request_data)
+                        sys.exit(0)
 
-            dataid,sold_out_at,revise_price,created_at = goodsInfo
+            if heat < 2:
+                goodsDb = MysqlDyLiveCommodity()
+                pix = str(uid)[-1]
+                goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
 
-            if revise_price is not None:
-                sys.exit(0)
+                dataid,sold_out_at,revise_price,created_at = goodsInfo
 
-            if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
-                add_t = 300
-                news_time = int(time.time()) + add_t
-                next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
-                rds_list.push_middle_list(next_rds)
-            else :
-                add_t = 1200
-                news_time = int(time.time()) + add_t
-                next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
-                rds_list.push_last_list(next_rds)
-
-        if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False:
-            sys.exit(0)
+                if revise_price is not None:
+                    sys.exit(0)
 
-        data = json.dumps({
-            "data": commodity_detail,
-            "extra": {
-                'room_id': room_id,
-                'promotion_id': promotion_id,
-                'product_id': product_id,
-                'uid': uid,
-                'heat': heat
-            }
-        })
-
-        rds_list.record_v1_score(1)
-        rds_list.push_v1_data_list(data)
-    except Exception as e:
-        rds_list.record_v1_score(0)
-        rds_list.add_request_params(request_data, heat)
-        PrintLog.print(
-            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常！' + str(e) + '\n'
-            + request_data
-        )
+                if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
+                    add_t = 300
+                    news_time = int(time.time()) + add_t
+                    next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
+                    rds_list.push_middle_list(next_rds)
+                else :
+                    add_t = 1200
+                    news_time = int(time.time()) + add_t
+                    next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time) 
+                    rds_list.push_last_list(next_rds)
+
+            if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False:
+                sys.exit(0)
+
+            data = json.dumps({
+                "data": commodity_detail,
+                "extra": {
+                    'room_id': room_id,
+                    'promotion_id': promotion_id,
+                    'product_id': product_id,
+                    'uid': uid,
+                    'heat': heat
+                }
+            })
+
+            rds_list.record_v1_score(1)
+            rds_list.push_v1_data_list(data)
+        except Exception as e:
+            rds_list.record_v1_score(0)
+            rds_list.add_request_params(request_data, heat)
+            PrintLog.print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常！' + str(e) + '\n'
+                + request_data
+            )
 
-    sys.exit(0)
 
 
 if __name__ == "__main__":
@@ -129,19 +135,12 @@ if __name__ == "__main__":
 
         increment = threading_count - active_count
 
-        while increment > 0:
+        if increment > 0:
 
             sys.stdout.flush()
 
-            request_data = rds.get_request_params(heat)
-
-            if request_data is None:
-                time.sleep(0.1)
-                break
-
-            task = threading.Thread(target=scrape, args=(request_data, heat))
+            task = threading.Thread(target=scrape, args=(heat))
             task.start()  # 准备就绪,等待cpu执行
-            increment = increment - 1
 
         current_time = time.time()
 

live_promotions_scraper.py

@@ -14,58 +14,65 @@ from log.print_log import PrintLog
 start_time = time.time()
 
 
-def scrape(heat, request_data_str):
+def scrape(heat):
 
-    rds_list = RdsLivePromotionsRequestList()
+    while True:
+        rds = RdsLivePromotionsRequestList()
+        request_data_str = rds.get_request_params(heat)
 
-    request_data = json.loads(request_data_str)
-    uid = request_data.get('uid')
-    room_id = request_data.get('room_id')
+        if request_data_str is None:
+            time.sleep(0.1)
+            break
 
-    if (uid is None) or (room_id is None):
-        PrintLog.print(
-            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求数据数据异常！' + '\n'
-            + request_data_str
-        )
-        sys.exit(0)
+        rds_list = RdsLivePromotionsRequestList()
 
-    room_id = str(room_id)
-    uid = str(uid)
-    PrintLog.print(time.strftime("%H:%M:%S", time.localtime()) + ' ' + str(heat) + ' ' + room_id + ' ' + uid)
+        request_data = json.loads(request_data_str)
+        uid = request_data.get('uid')
+        room_id = request_data.get('room_id')
 
-    try:
+        if (uid is None) or (room_id is None):
+            PrintLog.print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求数据数据异常！' + '\n'
+                + request_data_str
+            )
+            sys.exit(0)
 
-        response = LivePromotions.get_data(uid, room_id)
+        room_id = str(room_id)
+        uid = str(uid)
+        PrintLog.print(time.strftime("%H:%M:%S", time.localtime()) + ' ' + str(heat) + ' ' + room_id + ' ' + uid)
 
-        response_json = response.json()
+        try:
 
-        if response_json.get('promotions') is None:
-            # rds_list.record_score(0)
-            sys.exit(0)
+            response = LivePromotions.get_data(uid, room_id)
 
-        # 没有商品
-        if len(response_json.get('promotions')) == 0:
-            sys.exit(0)
+            response_json = response.json()
+
+            if response_json.get('promotions') is None:
+                # rds_list.record_score(0)
+                sys.exit(0)
 
-        data = json.dumps({
-            "data": response_json,
-            "extra": {
-                'room_id': room_id,
-                'uid': uid,
-                'heat': heat,
-            }
-        })
+            # 没有商品
+            if len(response_json.get('promotions')) == 0:
+                sys.exit(0)
 
-        rds_list.record_score(1)
-        rds_list.push_data_list(data)
-    except Exception as e:
-        rds_list.record_score(0)
-        PrintLog.print(
-            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常！' + str(e) + '\n'
-            + request_data_str
-        )
+            data = json.dumps({
+                "data": response_json,
+                "extra": {
+                    'room_id': room_id,
+                    'uid': uid,
+                    'heat': heat,
+                }
+            })
+
+            rds_list.record_score(1)
+            rds_list.push_data_list(data)
+        except Exception as e:
+            rds_list.record_score(0)
+            PrintLog.print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常！' + str(e) + '\n'
+                + request_data_str
+            )
 
-    sys.exit(0)
 
 
 if __name__ == "__main__":
@@ -87,19 +94,11 @@ if __name__ == "__main__":
 
         increment = threading_count - active_count
 
-        while increment > 0:
+        if increment > 0:
 
             sys.stdout.flush()
-
-            request_data_str = rds.get_request_params(heat)
-
-            if request_data_str is None:
-                time.sleep(0.1)
-                break
-
-            task = threading.Thread(target=scrape, args=(heat, request_data_str))
+            task = threading.Thread(target=scrape, args=(heat))
             task.start()  # 准备就绪,等待cpu执行
-            increment = increment - 1
 
         current_time = time.time()