|
@@ -15,99 +15,105 @@ from log.print_log import PrintLog
|
15
|
15
|
|
16
|
16
|
start_time = time.time()
|
17
|
17
|
|
18
|
|
-def scrape(request_data, heat):
|
19
|
|
- rds_list = RdsLiveCommodityDetailRequestList()
|
20
|
|
- request_split = request_data.split('@')
|
|
18
|
+def scrape(heat):
|
|
19
|
+ while True:
|
|
20
|
+ rds = RdsLiveCommodityDetailRequestList()
|
|
21
|
+ request_data = rds.get_request_params(heat)
|
|
22
|
+ if request_data is None:
|
|
23
|
+ time.sleep(0.1)
|
|
24
|
+ break
|
21
|
25
|
|
22
|
|
- if len(request_split) < 5:
|
23
|
|
- print('数据格式不对' + request_data)
|
24
|
|
- sys.exit(0)
|
|
26
|
+ rds_list = RdsLiveCommodityDetailRequestList()
|
|
27
|
+ request_split = request_data.split('@')
|
25
|
28
|
|
26
|
|
- promotion_id = request_split[0]
|
27
|
|
- product_id = request_split[1]
|
28
|
|
- uid = request_split[2]
|
29
|
|
- room_id = request_split[3]
|
30
|
|
- exec_time = int(request_split[4])
|
|
29
|
+ if len(request_split) < 5:
|
|
30
|
+ print('数据格式不对' + request_data)
|
|
31
|
+ sys.exit(0)
|
31
|
32
|
|
32
|
|
- if int(time.time()) < exec_time:
|
33
|
|
- sp_ns = exec_time - int(time.time())
|
34
|
|
- time.sleep(sp_ns)
|
|
33
|
+ promotion_id = request_split[0]
|
|
34
|
+ product_id = request_split[1]
|
|
35
|
+ uid = request_split[2]
|
|
36
|
+ room_id = request_split[3]
|
|
37
|
+ exec_time = int(request_split[4])
|
35
|
38
|
|
|
39
|
+ if int(time.time()) < exec_time:
|
|
40
|
+ sp_ns = exec_time - int(time.time())
|
|
41
|
+ time.sleep(sp_ns)
|
36
|
42
|
|
37
|
|
- PrintLog.print(
|
38
|
|
- time.strftime("%H:%M:%S", time.localtime()) + ' '
|
39
|
|
- + time.strftime("%H:%M:%S", time.localtime(exec_time)) + ' '
|
40
|
|
- + str(heat) + ' ' + promotion_id
|
41
|
|
- )
|
42
|
43
|
|
43
|
|
- try:
|
|
44
|
+ PrintLog.print(
|
|
45
|
+ time.strftime("%H:%M:%S", time.localtime()) + ' '
|
|
46
|
+ + time.strftime("%H:%M:%S", time.localtime(exec_time)) + ' '
|
|
47
|
+ + str(heat) + ' ' + promotion_id
|
|
48
|
+ )
|
44
|
49
|
|
45
|
|
- commodity_detail = LiveCommodityDetailV1.get_data(product_id)
|
|
50
|
+ try:
|
46
|
51
|
|
47
|
|
- if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False :
|
48
|
|
- rds_list.record_v1_score(0)
|
49
|
|
- PrintLog.print(
|
50
|
|
- time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 数据为空 ' + promotion_id
|
51
|
|
- )
|
|
52
|
+ commodity_detail = LiveCommodityDetailV1.get_data(product_id)
|
52
|
53
|
|
53
|
|
- if heat==0:
|
54
|
|
- if random.randint(0, 100) > 0:
|
55
|
|
- rds_list.add_request_params(request_data, heat)
|
56
|
|
- print('首次失败重新插入 ' + request_data)
|
57
|
|
- sys.exit(0)
|
|
54
|
+ if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False :
|
|
55
|
+ rds_list.record_v1_score(0)
|
|
56
|
+ PrintLog.print(
|
|
57
|
+ time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 数据为空 ' + promotion_id
|
|
58
|
+ )
|
58
|
59
|
|
59
|
|
- if heat>0:
|
60
|
|
- if random.randint(0, 9) > 0:
|
61
|
|
- rds_list.add_request_params(request_data, heat)
|
62
|
|
- print(str(heat) + '非首次失败重新插入 ' + request_data)
|
63
|
|
- sys.exit(0)
|
|
60
|
+ if heat==0:
|
|
61
|
+ if random.randint(0, 100) > 0:
|
|
62
|
+ rds_list.add_request_params(request_data, heat)
|
|
63
|
+ print('首次失败重新插入 ' + request_data)
|
|
64
|
+ sys.exit(0)
|
64
|
65
|
|
65
|
|
- if heat < 2:
|
66
|
|
- goodsDb = MysqlDyLiveCommodity()
|
67
|
|
- pix = str(uid)[-1]
|
68
|
|
- goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
|
|
66
|
+ if heat>0:
|
|
67
|
+ if random.randint(0, 9) > 0:
|
|
68
|
+ rds_list.add_request_params(request_data, heat)
|
|
69
|
+ print(str(heat) + '非首次失败重新插入 ' + request_data)
|
|
70
|
+ sys.exit(0)
|
69
|
71
|
|
70
|
|
- dataid,sold_out_at,revise_price,created_at = goodsInfo
|
|
72
|
+ if heat < 2:
|
|
73
|
+ goodsDb = MysqlDyLiveCommodity()
|
|
74
|
+ pix = str(uid)[-1]
|
|
75
|
+ goodsInfo = goodsDb.get_goods_info(room_id, product_id, pix)
|
71
|
76
|
|
72
|
|
- if revise_price is not None:
|
73
|
|
- sys.exit(0)
|
|
77
|
+ dataid,sold_out_at,revise_price,created_at = goodsInfo
|
74
|
78
|
|
75
|
|
- if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
|
76
|
|
- add_t = 300
|
77
|
|
- news_time = int(time.time()) + add_t
|
78
|
|
- next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time)
|
79
|
|
- rds_list.push_middle_list(next_rds)
|
80
|
|
- else :
|
81
|
|
- add_t = 1200
|
82
|
|
- news_time = int(time.time()) + add_t
|
83
|
|
- next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time)
|
84
|
|
- rds_list.push_last_list(next_rds)
|
85
|
|
-
|
86
|
|
- if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False:
|
87
|
|
- sys.exit(0)
|
|
79
|
+ if revise_price is not None:
|
|
80
|
+ sys.exit(0)
|
88
|
81
|
|
89
|
|
- data = json.dumps({
|
90
|
|
- "data": commodity_detail,
|
91
|
|
- "extra": {
|
92
|
|
- 'room_id': room_id,
|
93
|
|
- 'promotion_id': promotion_id,
|
94
|
|
- 'product_id': product_id,
|
95
|
|
- 'uid': uid,
|
96
|
|
- 'heat': heat
|
97
|
|
- }
|
98
|
|
- })
|
99
|
|
-
|
100
|
|
- rds_list.record_v1_score(1)
|
101
|
|
- rds_list.push_v1_data_list(data)
|
102
|
|
- except Exception as e:
|
103
|
|
- rds_list.record_v1_score(0)
|
104
|
|
- rds_list.add_request_params(request_data, heat)
|
105
|
|
- PrintLog.print(
|
106
|
|
- time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常!' + str(e) + '\n'
|
107
|
|
- + request_data
|
108
|
|
- )
|
|
82
|
+ if sold_out_at == '0000-00-00 00:00:00' or sold_out_at == '1970-01-01 08:00:00':
|
|
83
|
+ add_t = 300
|
|
84
|
+ news_time = int(time.time()) + add_t
|
|
85
|
+ next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time)
|
|
86
|
+ rds_list.push_middle_list(next_rds)
|
|
87
|
+ else :
|
|
88
|
+ add_t = 1200
|
|
89
|
+ news_time = int(time.time()) + add_t
|
|
90
|
+ next_rds = promotion_id + '@' + product_id + '@' + uid + '@' + room_id + '@' + str(news_time)
|
|
91
|
+ rds_list.push_last_list(next_rds)
|
|
92
|
+
|
|
93
|
+ if (commodity_detail is None) or (commodity_detail == '') or ('real_sell_num' in commodity_detail)==False:
|
|
94
|
+ sys.exit(0)
|
|
95
|
+
|
|
96
|
+ data = json.dumps({
|
|
97
|
+ "data": commodity_detail,
|
|
98
|
+ "extra": {
|
|
99
|
+ 'room_id': room_id,
|
|
100
|
+ 'promotion_id': promotion_id,
|
|
101
|
+ 'product_id': product_id,
|
|
102
|
+ 'uid': uid,
|
|
103
|
+ 'heat': heat
|
|
104
|
+ }
|
|
105
|
+ })
|
|
106
|
+
|
|
107
|
+ rds_list.record_v1_score(1)
|
|
108
|
+ rds_list.push_v1_data_list(data)
|
|
109
|
+ except Exception as e:
|
|
110
|
+ rds_list.record_v1_score(0)
|
|
111
|
+ rds_list.add_request_params(request_data, heat)
|
|
112
|
+ PrintLog.print(
|
|
113
|
+ time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '抛出异常!' + str(e) + '\n'
|
|
114
|
+ + request_data
|
|
115
|
+ )
|
109
|
116
|
|
110
|
|
- sys.exit(0)
|
111
|
117
|
|
112
|
118
|
|
113
|
119
|
if __name__ == "__main__":
|
|
@@ -129,19 +135,12 @@ if __name__ == "__main__":
|
129
|
135
|
|
130
|
136
|
increment = threading_count - active_count
|
131
|
137
|
|
132
|
|
- while increment > 0:
|
|
138
|
+ if increment > 0:
|
133
|
139
|
|
134
|
140
|
sys.stdout.flush()
|
135
|
141
|
|
136
|
|
- request_data = rds.get_request_params(heat)
|
137
|
|
-
|
138
|
|
- if request_data is None:
|
139
|
|
- time.sleep(0.1)
|
140
|
|
- break
|
141
|
|
-
|
142
|
|
- task = threading.Thread(target=scrape, args=(request_data, heat))
|
|
142
|
+ task = threading.Thread(target=scrape, args=(heat))
|
143
|
143
|
task.start() # 准备就绪,等待cpu执行
|
144
|
|
- increment = increment - 1
|
145
|
144
|
|
146
|
145
|
current_time = time.time()
|
147
|
146
|
|