更新脚本

douyin_user_scraper.py

@@ -6,10 +6,7 @@ import uuid
 import json
 from libs.aesgzip import tt_encrypt
 from xlog03 import *
-import sys
 from rds_model.db_redis import DbRedis
-from log.print_log import PrintLog
-from urllib.request import ProxyHandler,build_opener
 
 def get_mc():
     def a():
@@ -44,42 +41,11 @@ def get_random(i, random_type=1):
         salt = ''.join(sa)
         return salt
 
-
-def get_random_brand_type():
-    brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00'
-    return brand_type
-
-def get_scraper_proxy(proxy_key):
-    proxy_dict = redisModel.hgetall(proxy_key)
-    if (proxy_dict is None) or (len(proxy_dict) == 0):
-        return
-    proxy_list = list(proxy_dict)
-
-    now = int(time.time())
-    while True:
-        proxy = random.choice(proxy_list)
-        if proxy is None:
-            return
-        proxy_info = proxy_dict.get(proxy)
-        if proxy_info is None:
-            continue
-        proxy_info = json.loads(proxy_info)
-        expire_at = int(proxy_info.get('min_expired_at'))
-        # 删除过期的代理
-        if expire_at <= now:
-            redisModel.hdel(proxy_key, proxy)
-            proxy_list.remove(proxy)
-            continue
-
-        return proxy
-
 V1 = '6'
 V2 = '8'
 channel = 'huawei'
 device_type = 'RKK-YZ00'
 device_brand = 'HUAWEI'
-# redisModel = DbRedis.douyin_connect()
-redisModel = DbRedis.connect()
 
 print(channel, device_type)
 
@@ -273,18 +239,6 @@ class DouYinApi:
         douyin_url = douyin_url + "&_rticket=" + str(int(round(time.time() * 1000))) + "&ts=" + str(int(time.time()))
         return douyin_url
 
-    def get_user_info(self, user_id):
-        """获取用户信息
-        :param user_id: 用户ID
-        :return:
-        """
-        params = {
-            'user_id': user_id
-        }
-
-        douyin_url = 'https://aweme-eagle.snssdk.com/aweme/v1/user/?'
-        return self.__http_get(douyin_url, params)
-
     def __http_get(self, url, query_params=None):
         if query_params is None:
             query_params = {}
@@ -339,110 +293,14 @@ class DouYinApi:
         headers.update(sign)
         return headers
 
+    def get_user_info(self, user_id):
+        """获取用户信息
+        :param user_id: 用户ID
+        :return:
+        """
+        params = {
+            'user_id': user_id
+        }
 
-
-if __name__ == '__main__':
-    import warnings
-    warnings.filterwarnings("ignore")
-    param = str(sys.argv[1])
-
-    rkey = 'DOUYIN_USER_REGISTER_QUEUE'
-    pKey = 'IpProxyHashDaiLiYun'
-    uKey = 'DOUYIN_USER_SCRAPER_QUEUE'
-    sKey = 'DOUYIN_USER_INTODB_QUEUE'
-
-    cur_time = int(time.time())
-    if param == 'register':
-        print('===========注册程序===========')
-        PrintLog.print('===========注册程序===========')
-        while True:
-            now_time = int(time.time())
-            if (now_time - cur_time) > 270:
-                break
-
-            rltime = time.strftime("%H:%M:%S", time.localtime())
-            try:
-                llen = redisModel.llen(rkey)
-                if llen < 200:
-                    proxies = {
-                        "http": "http://lum-customer-c_44eb09ec-zone-data_center:4jme8s5hf9x6@zproxy.lum-superproxy.io:22225",
-                        "https": "http://lum-customer-c_44eb09ec-zone-data_center:4jme8s5hf9x6@zproxy.lum-superproxy.io:22225"
-                    }
-
-                    print(rltime + '注册代理:' + str(proxies))
-                    PrintLog.print(rltime + '注册代理:' + str(proxies))
-                    douApi = DouYinApi('', proxies)
-
-                    result = douApi.register_device()
-                    print(rltime + '结果:' + str(result))
-                    PrintLog.print(rltime + '结果:' + str(result))
-
-                    json_data = json.dumps(result)
-                    redisModel.lpush(rkey, json_data)
-                    print(rltime + '注册成功')
-                    PrintLog.print(rltime + '注册成功')
-                else:
-                    time.sleep(3)
-            except Exception as e:
-                print(rltime + '错误:' + str(e))
-                PrintLog.print(rltime + '错误:' + str(e))
-                continue
-
-    if param == 'scraper':
-        print('===========爬取程序===========')
-
-        while True:
-            now_time = int(time.time())
-            if (now_time - cur_time) > 270:
-                break
-
-            sltime = time.strftime("%H:%M:%S", time.localtime())
-            try:
-                proxy = get_scraper_proxy(pKey)
-                if proxy:
-                    proxies = {
-                        "http": "http://" + proxy,
-                        "https": "http://" + proxy
-                    }
-                else:
-                    time.sleep(2)
-                    continue
-
-                print(sltime + '爬取代理:' + str(proxies))
-                PrintLog.print(sltime + '爬取代理:' + str(proxies))
-                douApi = DouYinApi('', proxies)
-
-                rllen = redisModel.llen(rkey)
-                if rllen == 0:
-                    time.sleep(2)
-                    continue
-
-                json_data = redisModel.rpop(rkey)
-                dict_data = json.loads(json_data)
-                device_id, iid, udid, openudid, cookie = dict_data['device_id'], dict_data['iid'], dict_data['uuid'], \
-                                                         dict_data['openudid'], dict_data['cookie']
-                douApi.init_device_ids(device_id, iid, udid, openudid)
-
-                ullen = redisModel.llen(uKey)
-                if ullen == 0:
-                    time.sleep(2)
-                    continue
-
-                user_id = redisModel.rpop(uKey)
-                response = douApi.get_user_info(user_id)
-                if len(response.text) > 0 and response.json()['status_code'] == 0 and response.json()['user']:
-                    print(sltime + 'user_id:' + user_id + ' 爬取成功')
-                    PrintLog.print(sltime + 'user_id:' + user_id + ' 爬取成功')
-                    redisModel.lpush(sKey, response.text)
-                    redisModel.lpush(rkey, json_data)
-                else:
-                    print(sltime + 'user_id:' + user_id + ' 爬取失败')
-                    PrintLog.print(sltime + 'user_id:' + user_id + ' 爬取失败')
-                    scraper_time = dict_data['times']
-                    if scraper_time < 10:
-                        dict_data['times'] += 1
-                        redisModel.lpush(rkey, json.dumps(dict_data))
-            except Exception as e:
-                print(sltime + '错误:' + str(e))
-                PrintLog.print(sltime + '错误:' + str(e))
-                continue
+        douyin_url = 'https://aweme-eagle.snssdk.com/aweme/v1/user/?'
+        return self.__http_get(douyin_url, params)

douyin_hourly_ranklist_commerce_scraper.py

@@ -0,0 +1,282 @@
+from rds_model.rds_hourly_rank_list import RdsDouyinHourlyRankList
+import time
+import json
+import sys
+import threading
+import random
+import urllib
+import requests
+
+from libs.Xg04 import X_Gorgon
+from libs.proxy import Proxy
+from libs.mysql_user_living import *
+from libs.db_redis import DbRedis
+
+start_time = time.time()
+
+def get_random(i, random_type=1):
+    if random_type == 1:
+        return str(random.randint(1 * 10 ** (i - 1), 1 * 10 ** i - 1))
+    elif random_type == 8:
+        seed = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+    else:
+        seed = "1234567890abcde"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+def get_trace():
+    trace_list = [
+        '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
+        '00-ce7faf4409b7fcc0ae6135fdd4250468-ce7faf4409b7fcc0-01',
+        '00-ce7faf3b09b7fcc0ae6042f1d8100468-ce7faf3b09b7fcc0-01',
+        '00-cdd79d2309b7fcc0ae6625a4cb190468-cdd79d2309b7fcc0-01',
+        '00-cefde9f009b7fcc0ae6750e1349e0468-cefde9f009b7fcc0-01',
+        '00-ced2e6ef09b7fcc0ae67dd7bfe000468-ced2e6ef09b7fcc0-01',
+        '00-cefbfeb509b7fcc0ae659396a6ea0468-cefbfeb509b7fcc0-01',
+        '00-cefaa25409b7fcc0ae657726a3c30468-cefaa25409b7fcc0-01',
+    ]
+
+    return random.choice(trace_list)
+
+def get_random_brand_type():
+    brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00'
+    return brand_type
+
+def get_mc():
+    def a():
+        seed = "1234567890ABCDEF"
+        sa = []
+        for i in range(2):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+    k = ''
+    for i in range(6):
+        k += a() + ':'
+    return k[:-1]
+
+def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
+    
+    domain = 'webcast5-normal-c-lq.amemv.com'
+    url = 'https://' + domain + '/webcast/ranklist/hour/?'
+
+    rticket = str(int(time.time() * 1000))
+    mc = get_mc
+    udid = '8604' + get_random(11)
+    # openudid = '3b22' + str(udid.uuid4())[-12:]
+    ts = int(time.time())
+
+    trace_id = get_trace()
+    device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11]
+    
+    query = {
+        "style" : "3",
+        "hour_info" : "0",
+        "room_id" : room_id,
+        "rank_type" : "31",
+        "sec_anchor_id" : sec_anchor_id,
+        "webcast_sdk_version" : "1710",
+        "webcast_language" : "zh",
+        "webcast_locale" : "zh_CN",
+        "webcast_gps_access" : "2",
+        "os_api" : "23",
+        "device_type" : device_type,
+        "ssmix" : "a",
+        "manifest_version_code" : "130001",
+        "dpi" : "480",
+        "app_name" : "aweme",
+        "version_name" : "13.0.0",
+        "ts" : ts,
+        "cpu_support64" : "true",
+        "storage_type" : "0",
+        "app_type" : "normal",
+        "appTheme" : "dark",
+        "ac" : "wifi",
+        "host_abi" : "armeabi-v7a",
+        "update_version_code" : "13009900",
+        "channel" : "tengxun_new",
+        "_rticket" : rticket,
+        "device_platform" : "android",
+        "iid" : iid,
+        "version_code" : "130000",
+        "mac_address" : mc,
+        "cdid" : "81542dc6-2aca-4ff6-ac58-d94179e9d3e6",
+        "openudid" : openudid,
+        "device_id" : device_id,
+        "resolution" : "1080*1800",
+        "os_version" : "6.0",
+        "language" : "zh",
+        "device_brand" : device_brand,
+        "aid" : "1128"
+    }
+
+    query_params = urllib.parse.urlencode(query)
+    url = url + query_params
+
+    body = ''
+
+    xGorgon = X_Gorgon(query_params, body)
+
+    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
+        random.randint(1, 10))
+
+    headers = {
+        'Host': domain,
+        'Connection': 'keep-alive',
+        'Cache-Control': 'max-age=0',
+        'User-Agent': userAgent,
+        # 'accept-encoding': 'gzip, deflate',
+        "x-gorgon": xGorgon.get('X-Gorgon'),
+        "x-khronos": xGorgon.get('X-Khronos'),
+        'passport-sdk-version' : '18',
+        'sdk-version' : '2',
+        'x-ss-dp' : '1128',
+        'x-tt-trace-id' : trace_id,
+    }
+
+    retry = 0
+    response_json = None
+
+    while True:
+        if retry > 50:
+            break
+
+        retry += 1
+
+        proxy = Proxy.get()
+
+        proxies = {
+            "http": "http://" + proxy,
+            "https": "http://" + proxy
+        }
+
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                proxies=proxies,
+                timeout=8
+            )
+            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
+                response_json = response.json()
+
+                if (response_json.get('data') is not None):
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取成功！' 
+                    )
+                    break
+                else:
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取失败！' 
+                        + '\n'
+                        + str(response.text)
+                        + str(Proxy.proxy_info)
+                    )
+                
+            else:
+                print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 爬取http连接失败！' 
+                    + str(response.status_code) 
+                    + '\n'
+                    + str(Proxy.proxy_info)
+                    + '\n'
+                    + '爬取结果' + str(response)
+                    + '\n'
+                )
+                time.sleep(1)
+        except requests.exceptions.ProxyError as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 代理过期！' 
+                + str(e) 
+                + '\n'
+                + str(Proxy.proxy_info)
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except requests.exceptions.ConnectTimeout as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' ConnectTimeout！' 
+                + str(e) 
+                + '\n'
+                + str(Proxy.proxy_info)
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except Exception as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 请求抛出异常！' 
+                + str(e) 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            pass
+
+    return response_json
+
+def scrape(room_id,sec_anchor_id,anchor_id):
+    rds_list = RdsDouyinHourlyRankList()
+
+    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+    rdid = DbRedis.connect().rpop(key)
+    if rdid:
+        result = rdid.split('@@@')
+    else:
+        result = []
+        return None
+
+    DbRedis.connect().lpush(key, rdid)
+
+    try:
+        # 带货小时榜
+        commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
+        
+        if commerce_response_json is None:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 带货小时榜数据获取失败！响应数据为空！' 
+            )
+            sys.exit(0)
+        else:
+            data = json.dumps({
+                "data": commerce_response_json.get('data'),
+                "extra": {}
+            })
+            rds_list.push_commerce_data_list(data)
+
+    except Exception as e:
+        print(
+            time.strftime("%H:%M:%S", time.localtime()) 
+            + ' ' 
+            + '数据异常：' 
+            + str(e)
+        )
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    print("主方法开始执行")
+
+    room_info = MysqlUserLiving().get_living_info()
+
+    if room_info is None:
+        sys.exit(0)
+    
+    room_id,anchor_id,sec_anchor_id = room_info
+    
+    task = threading.Thread(target=scrape, args=(room_id, sec_anchor_id, anchor_id,))
+    task.start()  # 准备就绪, 等待cpu执行

douyin_hourly_ranklist_popularity_scraper.py

@@ -0,0 +1,292 @@
+from rds_model.rds_hourly_rank_list import RdsDouyinHourlyRankList
+import time
+import json
+import sys
+import threading
+import random
+import urllib
+import requests
+
+from libs.Xg04 import X_Gorgon
+from libs.proxy import Proxy
+from libs.mysql_user_living import *
+from libs.db_redis import DbRedis
+
+start_time = time.time()
+
+def get_random(i, random_type=1):
+    if random_type == 1:
+        return str(random.randint(1 * 10 ** (i - 1), 1 * 10 ** i - 1))
+    elif random_type == 8:
+        seed = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+    else:
+        seed = "1234567890abcde"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+def get_trace():
+    trace_list = [
+        '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
+        '00-ce7faf4409b7fcc0ae6135fdd4250468-ce7faf4409b7fcc0-01',
+        '00-ce7faf3b09b7fcc0ae6042f1d8100468-ce7faf3b09b7fcc0-01',
+        '00-cdd79d2309b7fcc0ae6625a4cb190468-cdd79d2309b7fcc0-01',
+        '00-cefde9f009b7fcc0ae6750e1349e0468-cefde9f009b7fcc0-01',
+        '00-ced2e6ef09b7fcc0ae67dd7bfe000468-ced2e6ef09b7fcc0-01',
+        '00-cefbfeb509b7fcc0ae659396a6ea0468-cefbfeb509b7fcc0-01',
+        '00-cefaa25409b7fcc0ae657726a3c30468-cefaa25409b7fcc0-01',
+    ]
+
+    return random.choice(trace_list)
+
+def get_random_brand_type():
+    brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00'
+    return brand_type
+
+def get_mc():
+    def a():
+        seed = "1234567890ABCDEF"
+        sa = []
+        for i in range(2):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+    k = ''
+    for i in range(6):
+        k += a() + ':'
+    return k[:-1]
+
+def get_popularity_rank_list_data(room_id, sec_anchor_id, anchor_id, result):
+    
+    domain = 'webcast5-normal-c-lq.amemv.com'
+    url = 'https://' + domain + '/webcast/ranklist/hour/?'
+
+    rticket = str(int(time.time() * 1000))
+    mc = get_mc
+    udid = '8604' + get_random(11)
+    ts = int(time.time())
+    # openudid = '3b22' + str(udid.uuid4())[-12:]
+
+    trace_id = get_trace()
+
+    device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11]
+    
+    query = {
+        "anchor_id" : anchor_id,
+        "room_id" : room_id,
+        "sec_anchor_id" : sec_anchor_id,
+        "sec_user_id" : "null",
+        "webcast_sdk_version" : "2010",
+        "webcast_language" : "zh",
+        "webcast_locale" : "zh_CN",
+        "webcast_gps_access" : "2",
+        "current_network_quality_info" : "%7B%22http_rtt%22%3A110%2C%22tcp_rtt%22%3A90%2C%22quic_rtt%22%3A90%2C%22downstream_throughput_kbps%22%3A8185%2C%22video_download_speed%22%3A411%2C%22quic_receive_loss_rate%22%3A-1%2C%22quic_send_loss_rate%22%3A-1%2C%22net_effective_connection_type%22%3A5%7D",
+        "os_api" : "23",
+        "device_type" : device_type,
+        "ssmix" : "a",
+        "manifest_version_code" : "160001",
+        "dpi" : "480",
+        "app_name" : "aweme",
+        "version_name" : "16.0.0",
+        "ts" : ts,
+        "cpu_support64" : "true",
+        "app_type" : "normal",
+        "appTheme" : "dark",
+        "ac" : "wifi",
+        "host_abi" : "armeabi-v7a",
+        "update_version_code" : "16009900",
+        "channel" : "wandoujia_lesi_1128_0507",
+        "_rticket" : rticket,
+        "device_platform" : "android",
+        "iid" : iid,
+        "version_code" : "160000",
+        "cdid" : "09e904ed-66a9-4e89-9661-afae7f61e6c5",
+        "openudid" : openudid,
+        "device_id" : device_id,
+        "resolution" : "1080*1800",
+        "os_version" : "6.0",
+        "language" : "zh",
+        "device_brand" : device_brand,
+        "aid" : "1128",
+        "minor_status" : "0"
+    }
+
+    query_params = urllib.parse.urlencode(query)
+    url = url + query_params
+
+    body = ''
+
+    xGorgon = X_Gorgon(query_params, body)
+
+    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
+        random.randint(1, 10))
+
+    headers = {
+        'Host': domain,
+        'Connection': 'keep-alive',
+        'Cache-Control': 'max-age=0',
+        'Upgrade-Insecure-Requests': '1',
+        'User-Agent': userAgent,
+        # 'accept-encoding': 'gzip, deflate',
+        "x-gorgon": xGorgon.get('X-Gorgon'),
+        "x-khronos": xGorgon.get('X-Khronos'),
+        'passport-sdk-version' : '18',
+        'sdk-version' : '2',
+        'x-ss-dp' : '1128',
+        'x-tt-trace-id' : trace_id,
+    }
+
+    retry = 0
+    response_json = None
+
+    while True:
+        if retry > 50:
+            break
+
+        retry += 1
+
+        proxy = Proxy.get()
+
+        proxies = {
+            "http": "http://" + proxy,
+            "https": "http://" + proxy
+        }
+
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                proxies=proxies,
+                timeout=8
+            )
+            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
+                response_json = response.json()
+
+                if (response_json.get('data') is not None):
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取成功！' 
+                        + '\n'
+                        + room_id 
+                    )
+                    break
+                else:
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取失败！' 
+                        + '\n'
+                        + room_id 
+                        + '\n'
+                        + response.text
+                        + Proxy.proxy_info
+                    )
+                
+            else:
+                print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 爬取http连接失败！' 
+                    + str(response.status_code) 
+                    + '\n'
+                    + Proxy.proxy_info 
+                    + '\n'
+                    + room_id 
+                    + '\n'
+                )
+                time.sleep(1)
+        except requests.exceptions.ProxyError as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 代理过期！' 
+                + str(e) 
+                + '\n'
+                + room_id 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except requests.exceptions.ConnectTimeout as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' ConnectTimeout！' 
+                + str(e) 
+                + '\n'
+                + room_id 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except Exception as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 请求抛出异常！' 
+                + str(e) 
+                + '\n'
+                + room_id 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            pass
+
+    return response_json
+
+def scrape(room_id,sec_anchor_id,anchor_id):
+    rds_list = RdsDouyinHourlyRankList()
+
+    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+    rdid = DbRedis.connect().rpop(key)
+    if rdid:
+        result = rdid.split('@@@')
+    else:
+        return None
+
+    DbRedis.connect().lpush(key, rdid)
+
+    try:
+        # 人气小时榜
+        popularity_response_json = get_popularity_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id)
+        
+        if popularity_response_json is None:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 人气小时榜数据获取失败！响应数据为空！' 
+            )
+            sys.exit(0)
+        else:
+            data = json.dumps({
+                "data": popularity_response_json.get('data'),
+                "extra": {}
+            })
+            rds_list.push_popularity_data_list(data)
+
+    except Exception as e:
+        print(
+            time.strftime("%H:%M:%S", time.localtime()) 
+            + ' ' 
+            + '数据异常：' 
+            + str(e)
+        )
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    print("主方法开始执行")
+
+    room_info = MysqlUserLiving().get_living_info()
+
+    if room_info is None:
+        sys.exit(0)
+    
+    room_id,anchor_id,sec_anchor_id = room_info
+    
+    task = threading.Thread(target=scrape, args=(room_id, sec_anchor_id, anchor_id,))
+    task.start()  # 准备就绪, 等待cpu执行

douyin_hourly_ranklist_scraper.py

@@ -1,653 +0,0 @@
-from rds_model.rds_hourly_rank_list import RdsDouyinHourlyRankList
-import time
-import json
-import sys
-import threading
-import random
-import urllib
-import requests
-
-from rds_model.db_redis import DbRedis
-from log.print_log import PrintLog
-from libs.Xg04 import X_Gorgon
-from libs.proxy import Proxy
-from libs.mysql_user_living import *
-
-start_time = time.time()
-
-def get_random(i, random_type=1):
-    if random_type == 1:
-        return str(random.randint(1 * 10 ** (i - 1), 1 * 10 ** i - 1))
-    elif random_type == 8:
-        seed = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-        sa = []
-        for i in range(i):
-            sa.append(random.choice(seed))
-        salt = ''.join(sa)
-        return salt
-    else:
-        seed = "1234567890abcde"
-        sa = []
-        for i in range(i):
-            sa.append(random.choice(seed))
-        salt = ''.join(sa)
-        return salt
-
-
-def get_random_brand_type():
-    brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00'
-    return brand_type
-
-def get_mc():
-    def a():
-        seed = "1234567890ABCDEF"
-        sa = []
-        for i in range(2):
-            sa.append(random.choice(seed))
-        salt = ''.join(sa)
-        return salt
-
-    k = ''
-    for i in range(6):
-        k += a() + ':'
-    return k[:-1]
-
-def get_whole_station_rank_data(room_id, sec_anchor_id, anchor_id):
-
-    domain = 'webcast5-normal-c-lq.amemv.com'
-    url = 'https://' + domain + '/webcast/ranklist/hour/?'
-
-    rticket = str(int(time.time() * 1000))
-    ts = int(time.time())
-    mc = get_mc
-    udid = '8604' + get_random(11)
-    query = {
-        "manifest_version_code" : "110001",
-        "dpi" : "480",
-        "app_name" : "aweme",
-        "version_name" : "11.0.0",
-        "ts" : ts,
-        "cpu_support64" : "true",
-        "app_type" : "normal",
-        "ac" : "wifi", 
-        "host_abi" : "armeabi-v7a",
-        "channel" : "wandoujia_aweme_feisuo",
-        "device_platform" : "android",
-        "iid" : "3932281687270606",
-        "version_code" : "110000",
-        "cdid" : "1d06013c-ff0b-427b-b1ab-6700259c15c6",
-        "openudid" : "291f3ce2efe59345",
-        "hour_info" : "0",
-        "room_id" : room_id,
-        "rank_type" : "12",
-        "sec_anchor_id" : sec_anchor_id,
-        "anchor_id" : anchor_id,
-        "webcast_sdk_version" : "1510",
-        "webcast_language" : "zh",
-        "webcast_locale" : "zh_CN",
-        "os_api" : "23",
-        "device_type" : "HUAWEI MLA-AL10",
-        "ssmix" : "a",
-        "update_version_code" : "11009900",
-        "cdid" : "1d06013c-ff0b-427b-b1ab-6700259c15c6",
-        "openudid" : "291f3ce2efe59345",
-        "device_id" : '49388718822',
-        "resolution" : "1080*1800",
-        "os_version" : "6.0",
-        "language" : "zh",
-        "device_brand" : "HUAWEI",
-        "_rticket" : rticket,
-        "aid" : "1128"
-    }
-
-    query_params = urllib.parse.urlencode(query)
-    url = url + query_params
-
-    body = ''
-
-    xGorgon = X_Gorgon(query_params, body)
-
-    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
-        random.randint(1, 10))
-
-    headers = {
-        'Host': domain,
-        'Connection': 'keep-alive',
-        'User-Agent': userAgent,
-        'accept-encoding': 'gzip, deflate',
-        "x-gorgon": xGorgon.get('X-Gorgon'),
-        "x-khronos": xGorgon.get('X-Khronos'),
-        'sdk-version' : '2',
-        'x-ss-dp' : '1128',
-        'x-tt-trace-id' : '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
-    }
-
-    retry = 0
-    response_json = None
-
-    while True:
-        if retry > 10:
-            break
-
-        retry += 1
-
-        proxy = Proxy.get()
-
-        proxies = {
-            "http": "http://" + proxy,
-            "https": "http://" + proxy
-        }
-
-        try:
-            response = requests.get(
-                url,
-                headers=headers,
-                proxies=proxies,
-                timeout=8
-            )
-            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
-                response_json = response.json()
-
-                if (response_json.get('data') is not None):
-                    break
-                else:
-                    print(
-                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                        + ' 数据获取失败！' 
-                        + '\n'
-                        + room_id 
-                        + '\n'
-                        + response.text
-                        + Proxy.proxy_info
-                    )
-                
-            else:
-                print(
-                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                    + ' 爬取http连接失败！' 
-                    + str(response.status_code) 
-                    + '\n'
-                    + Proxy.proxy_info 
-                    + '\n'
-                    + room_id 
-                    + '\n'
-                )
-                time.sleep(1)
-        except requests.exceptions.ProxyError as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 代理过期！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except requests.exceptions.ConnectTimeout as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' ConnectTimeout！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except Exception as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 请求抛出异常！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            pass
-
-    return response_json
-
-def get_commerce_rank_list_data(room_id, sec_anchor_id, anchor_id):
-    
-    domain = 'webcast5-normal-c-lq.amemv.com'
-    url = 'https://' + domain + '/webcast/ranklist/hour/?'
-
-    rticket = str(int(time.time() * 1000))
-    mc = get_mc
-    udid = '8604' + get_random(11)
-    # openudid = '3b22' + str(udid.uuid4())[-12:]
-    ts = int(time.time())
-    
-    query = {
-        "style" : "3",
-        "hour_info" : "0",
-        "room_id" : room_id,
-        "rank_type" : "31",
-        "sec_anchor_id" : sec_anchor_id,
-        "webcast_sdk_version" : "1710",
-        "webcast_language" : "zh",
-        "webcast_locale" : "zh_CN",
-        "webcast_gps_access" : "2",
-        "os_api" : "23",
-        "device_type" : "HUAWEI+MLA-AL10",
-        "ssmix" : "a",
-        "manifest_version_code" : "130001",
-        "dpi" : "480",
-        "app_name" : "aweme",
-        "version_name" : "13.0.0",
-        "ts" : ts,
-        "cpu_support64" : "true",
-        "storage_type" : "0",
-        "app_type" : "normal",
-        "appTheme" : "dark",
-        "ac" : "wifi",
-        "host_abi" : "armeabi-v7a",
-        "update_version_code" : "13009900",
-        "channel" : "tengxun_new",
-        "_rticket" : rticket,
-        "device_platform" : "android",
-        "iid" : "2876750595379005",
-        "version_code" : "130000",
-        "mac_address" : mc,
-        "cdid" : "81542dc6-2aca-4ff6-ac58-d94179e9d3e6",
-        "openudid" : "291f3ce2efe59345",
-        "device_id" : "49388718822",
-        "resolution" : "1080*1800",
-        "os_version" : "6.0",
-        "language" : "zh",
-        "device_brand" : "HUAWEI",
-        "aid" : "1128"
-    }
-
-    query_params = urllib.parse.urlencode(query)
-    url = url + query_params
-
-    body = ''
-
-    xGorgon = X_Gorgon(query_params, body)
-
-    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
-        random.randint(1, 10))
-
-    headers = {
-        'Host': domain,
-        'Connection': 'keep-alive',
-        'Cache-Control': 'max-age=0',
-        'User-Agent': userAgent,
-        'accept-encoding': 'gzip, deflate',
-        "x-gorgon": xGorgon.get('X-Gorgon'),
-        "x-khronos": xGorgon.get('X-Khronos'),
-        'passport-sdk-version' : '18',
-        'sdk-version' : '2',
-        'x-ss-dp' : '1128',
-        'x-tt-trace-id' : '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
-    }
-
-    retry = 0
-    response_json = None
-
-    while True:
-        if retry > 10:
-            break
-
-        retry += 1
-
-        proxy = Proxy.get()
-
-        proxies = {
-            "http": "http://" + proxy,
-            "https": "http://" + proxy
-        }
-
-        try:
-            response = requests.get(
-                url,
-                headers=headers,
-                proxies=proxies,
-                timeout=8
-            )
-            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
-                response_json = response.json()
-
-                if (response_json.get('data') is not None):
-                    break
-                else:
-                    print(
-                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                        + ' 数据获取失败！' 
-                        + '\n'
-                        + room_id 
-                        + '\n'
-                        + response.text
-                        + Proxy.proxy_info
-                    )
-                
-            else:
-                print(
-                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                    + ' 爬取http连接失败！' 
-                    + str(response.status_code) 
-                    + '\n'
-                    + Proxy.proxy_info 
-                    + '\n'
-                    + room_id 
-                    + '\n'
-                )
-                time.sleep(1)
-        except requests.exceptions.ProxyError as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 代理过期！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except requests.exceptions.ConnectTimeout as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' ConnectTimeout！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except Exception as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 请求抛出异常！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            pass
-
-    return response_json
-
-def get_popularity_rank_list_data(room_id, sec_anchor_id, anchor_id):
-    
-    domain = 'webcast5-normal-c-lq.amemv.com'
-    url = 'https://' + domain + '/webcast/ranklist/hour/?'
-
-    rticket = str(int(time.time() * 1000))
-    mc = get_mc
-    udid = '8604' + get_random(11)
-    ts = int(time.time())
-    # openudid = '3b22' + str(udid.uuid4())[-12:]
-    
-    query = {
-        "anchor_id" : anchor_id,
-        "room_id" : room_id,
-        "sec_anchor_id" : sec_anchor_id,
-        "sec_user_id" : "null",
-        "webcast_sdk_version" : "2010",
-        "webcast_language" : "zh",
-        "webcast_locale" : "zh_CN",
-        "webcast_gps_access" : "2",
-        "current_network_quality_info" : "%7B%22http_rtt%22%3A110%2C%22tcp_rtt%22%3A90%2C%22quic_rtt%22%3A90%2C%22downstream_throughput_kbps%22%3A8185%2C%22video_download_speed%22%3A411%2C%22quic_receive_loss_rate%22%3A-1%2C%22quic_send_loss_rate%22%3A-1%2C%22net_effective_connection_type%22%3A5%7D",
-        "os_api" : "23",
-        "device_type" : "HUAWEI+MLA-AL10",
-        "ssmix" : "a",
-        "manifest_version_code" : "160001",
-        "dpi" : "480",
-        "app_name" : "aweme",
-        "version_name" : "16.0.0",
-        "ts" : ts,
-        "cpu_support64" : "true",
-        "app_type" : "normal",
-        "appTheme" : "dark",
-        "ac" : "wifi",
-        "host_abi" : "armeabi-v7a",
-        "update_version_code" : "16009900",
-        "channel" : "wandoujia_lesi_1128_0507",
-        "_rticket" : rticket,
-        "device_platform" : "android",
-        "iid" : "273107070769192",
-        "version_code" : "160000",
-        "cdid" : "09e904ed-66a9-4e89-9661-afae7f61e6c5",
-        "openudid" : "291f3ce2efe59345",
-        "device_id" : "49388718822",
-        "resolution" : "1080*1800",
-        "os_version" : "6.0",
-        "language" : "zh",
-        "device_brand" : "HUAWEI",
-        "aid" : "1128",
-        "minor_status" : "0"
-    }
-
-    query_params = urllib.parse.urlencode(query)
-    url = url + query_params
-
-    body = ''
-
-    xGorgon = X_Gorgon(query_params, body)
-
-    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
-        random.randint(1, 10))
-
-    headers = {
-        'Host': domain,
-        'Connection': 'keep-alive',
-        'Cache-Control': 'max-age=0',
-        'Upgrade-Insecure-Requests': '1',
-        'User-Agent': userAgent,
-        'accept-encoding': 'gzip, deflate',
-        "x-gorgon": xGorgon.get('X-Gorgon'),
-        "x-khronos": xGorgon.get('X-Khronos'),
-        'passport-sdk-version' : '18',
-        'sdk-version' : '2',
-        'x-ss-dp' : '1128',
-        'x-tt-trace-id' : '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
-    }
-
-    retry = 0
-    response_json = None
-
-    while True:
-        if retry > 10:
-            break
-
-        retry += 1
-
-        proxy = Proxy.get()
-
-        proxies = {
-            "http": "http://" + proxy,
-            "https": "http://" + proxy
-        }
-
-        try:
-            response = requests.get(
-                url,
-                headers=headers,
-                proxies=proxies,
-                timeout=8
-            )
-            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
-                response_json = response.json()
-
-                if (response_json.get('data') is not None):
-                    break
-                else:
-                    print(
-                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                        + ' 数据获取失败！' 
-                        + '\n'
-                        + room_id 
-                        + '\n'
-                        + response.text
-                        + Proxy.proxy_info
-                    )
-                
-            else:
-                print(
-                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                    + ' 爬取http连接失败！' 
-                    + str(response.status_code) 
-                    + '\n'
-                    + Proxy.proxy_info 
-                    + '\n'
-                    + room_id 
-                    + '\n'
-                )
-                time.sleep(1)
-        except requests.exceptions.ProxyError as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 代理过期！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except requests.exceptions.ConnectTimeout as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' ConnectTimeout！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            Proxy.del_proxy(proxy)
-            pass
-        except Exception as e:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 请求抛出异常！' 
-                + str(e) 
-                + '\n'
-                + room_id 
-                + '\n'
-                + Proxy.proxy_info
-            )
-            pass
-
-    return response_json
-
-def scrape(room_id,sec_anchor_id,anchor_id):
-    rds_list = RdsDouyinHourlyRankList()
-    time.sleep(0.1)
-
-    try:
-        # 全站小时榜
-        whole_station_response_json = get_whole_station_rank_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id)
-        # 带货小时榜
-        commerce_response_json = get_commerce_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id)
-        # 人气小时榜
-        popularity_response_json = get_popularity_rank_list_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id)
-
-       
-        if whole_station_response_json is None:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 全站小时榜数据获取失败！响应数据为空！' 
-            )
-            sys.exit(0)
-        else:
-            data = json.dumps({
-                "data": whole_station_response_json.get('data'),
-                "extra": {}
-            })
-            rds_list.push_whole_station_data_list(data)
-        
-        if commerce_response_json is None:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 带货小时榜数据获取失败！响应数据为空！' 
-            )
-            sys.exit(0)
-        else:
-            data = json.dumps({
-                "data": commerce_response_json.get('data'),
-                "extra": {}
-            })
-            rds_list.push_commerce_data_list(data)
-        
-        if popularity_response_json is None:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 人气小时榜数据获取失败！响应数据为空！' 
-            )
-            sys.exit(0)
-        else:
-            data = json.dumps({
-                "data": popularity_response_json.get('data'),
-                "extra": {}
-            })
-            rds_list.push_popularity_data_list(data)
-
-    except Exception as e:
-        print(
-            time.strftime("%H:%M:%S", time.localtime()) 
-            + ' ' 
-            + '数据异常：' 
-            + str(e)
-        )
-
-    sys.exit(0)
-
-if __name__ == "__main__":
-    print("主方法开始执行")
-    # 并行线程数
-    threading_count = int(sys.argv[1])
-
-    rds = RdsDouyinHourlyRankList()
-
-    print(
-        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-        + ' ' 
-        + ' 开始执行，更新直播队列长度：' 
-        + str(rds.get_len())
-    )
-
-    while True:
-        sys.stdout.flush()
-        
-        # 减去主线程
-        active_count = threading.active_count() - 1
-       
-        increment = threading_count - active_count
-        
-        while increment > 0:
-            sys.stdout.flush()
-
-            room_info = MysqlUserLiving().get_living_info()
-            room_id = False
-
-            if room_info is None:
-                time.sleep(60)
-                continue
-            
-            room_id,anchor_id,sec_anchor_id = room_info
-           
-            task = threading.Thread(target=scrape, args=(room_id, sec_anchor_id, anchor_id,))
-            task.start()  # 准备就绪, 等待cpu执行
-            increment = increment - 1
-           
-        current_time = time.time()
-
-        if current_time - start_time > 300:
-            print(
-                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                + ' 主方法执行终止'
-            )
-            sys.exit(0)
-
-        time.sleep(0.01)

douyin_hourly_ranklist_whole_scraper.py

@@ -0,0 +1,275 @@
+from rds_model.rds_hourly_rank_list import RdsDouyinHourlyRankList
+import time
+import json
+import sys
+import threading
+import random
+import urllib
+import requests
+
+from libs.Xg04 import X_Gorgon
+from libs.proxy import Proxy
+from libs.mysql_user_living import *
+from libs.db_redis import DbRedis
+
+start_time = time.time()
+
+def get_random(i, random_type=1):
+    if random_type == 1:
+        return str(random.randint(1 * 10 ** (i - 1), 1 * 10 ** i - 1))
+    elif random_type == 8:
+        seed = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+    else:
+        seed = "1234567890abcde"
+        sa = []
+        for i in range(i):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+
+def get_random_brand_type():
+    brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00'
+    return brand_type
+
+def get_mc():
+    def a():
+        seed = "1234567890ABCDEF"
+        sa = []
+        for i in range(2):
+            sa.append(random.choice(seed))
+        salt = ''.join(sa)
+        return salt
+
+    k = ''
+    for i in range(6):
+        k += a() + ':'
+    return k[:-1]
+
+def get_trace():
+    trace_list = [
+        '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
+        '00-ce7faf4409b7fcc0ae6135fdd4250468-ce7faf4409b7fcc0-01',
+        '00-ce7faf3b09b7fcc0ae6042f1d8100468-ce7faf3b09b7fcc0-01',
+        '00-cdd79d2309b7fcc0ae6625a4cb190468-cdd79d2309b7fcc0-01',
+        '00-cefde9f009b7fcc0ae6750e1349e0468-cefde9f009b7fcc0-01',
+        '00-ced2e6ef09b7fcc0ae67dd7bfe000468-ced2e6ef09b7fcc0-01',
+        '00-cefbfeb509b7fcc0ae659396a6ea0468-cefbfeb509b7fcc0-01',
+        '00-cefaa25409b7fcc0ae657726a3c30468-cefaa25409b7fcc0-01',
+    ]
+
+    return random.choice(trace_list)
+
+def get_whole_station_rank_data(room_id, sec_anchor_id, anchor_id, result):
+
+    domain = 'webcast5-normal-c-lq.amemv.com'
+    url = 'https://' + domain + '/webcast/ranklist/hour/?'
+
+    rticket = str(int(time.time() * 1000))
+    ts = int(time.time())
+    mc = get_mc
+    udid = '8604' + get_random(11)
+    trace_id = get_trace()
+
+    device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11]
+
+    query = {
+        "manifest_version_code" : "110001",
+        "dpi" : "480",
+        "app_name" : "aweme",
+        "version_name" : "11.0.0",
+        "ts" : ts,
+        "cpu_support64" : "true",
+        "app_type" : "normal",
+        "ac" : "wifi", 
+        "host_abi" : "armeabi-v7a",
+        "channel" : "wandoujia_aweme_feisuo",
+        "device_platform" : "android",
+        "iid" : iid,
+        "version_code" : "110000",
+        "cdid" : "1d06013c-ff0b-427b-b1ab-6700259c15c6",
+        "hour_info" : "0",
+        "room_id" : room_id,
+        "rank_type" : "12",
+        "sec_anchor_id" : sec_anchor_id,
+        "anchor_id" : anchor_id,
+        "webcast_sdk_version" : "1510",
+        "webcast_language" : "zh",
+        "webcast_locale" : "zh_CN",
+        "os_api" : "23",
+        "device_type" : device_type,
+        "ssmix" : "a",
+        "update_version_code" : "11009900",
+        "openudid" : openudid,
+        "device_id" : device_id,
+        "resolution" : "1080*1800",
+        "os_version" : "6.0",
+        "language" : "zh",
+        "device_brand" : device_brand,
+        "_rticket" : rticket,
+        "aid" : "1128"
+    }
+
+    query_params = urllib.parse.urlencode(query)
+    url = url + query_params
+
+    body = ''
+
+    xGorgon = X_Gorgon(query_params, body)
+
+    userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
+        random.randint(1, 10))
+
+    headers = {
+        'Host': domain,
+        'Connection': 'keep-alive',
+        'User-Agent': userAgent,
+        # 'accept-encoding': 'gzip, deflate',
+        "x-gorgon": xGorgon.get('X-Gorgon'),
+        "x-khronos": xGorgon.get('X-Khronos'),
+        'sdk-version' : '2',
+        'x-ss-dp' : '1128',
+        'x-tt-trace-id' : trace_id
+    }
+
+    retry = 0
+    response_json = None
+
+    while True:
+        if retry > 50:
+            break
+
+        retry += 1
+
+        proxy = Proxy.get()
+
+        proxies = {
+            "http": "http://" + proxy,
+            "https": "http://" + proxy
+        }
+
+        try:
+            response = requests.get(
+                url,
+                headers=headers,
+                proxies=proxies,
+                timeout=8
+            )
+            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
+                response_json = response.json()
+
+                if (response_json.get('data') is not None):
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取成功！' 
+                    )
+
+                    break
+                else:
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取失败！' 
+                        + '\n'
+                        + response.text
+                        + Proxy.proxy_info
+                    )
+                
+            else:
+                print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                    + ' 爬取http连接失败！' 
+                    + str(response.status_code) 
+                    + '\n'
+                    + Proxy.proxy_info 
+                    + '\n'
+                )
+                time.sleep(1)
+        except requests.exceptions.ProxyError as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 代理过期！' 
+                + str(e) 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except requests.exceptions.ConnectTimeout as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' ConnectTimeout！' 
+                + str(e) 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            Proxy.del_proxy(proxy)
+            pass
+        except Exception as e:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 请求抛出异常！' 
+                + str(e) 
+                + '\n'
+                + Proxy.proxy_info
+            )
+            pass
+
+    return response_json
+
+def scrape(room_id,sec_anchor_id,anchor_id):
+    rds_list = RdsDouyinHourlyRankList()
+
+    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+    rdid = DbRedis.connect().rpop(key)
+    if rdid:
+        result = rdid.split('@@@')
+    else:
+        result = []
+        return None
+
+    DbRedis.connect().lpush(key, rdid)
+
+    try:
+        # 全站小时榜
+        whole_station_response_json = get_whole_station_rank_data(room_id=room_id, sec_anchor_id=sec_anchor_id, anchor_id=anchor_id, result=result)
+       
+        if whole_station_response_json is None:
+            print(
+                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                + ' 全站小时榜数据获取失败！响应数据为空！' 
+            )
+            sys.exit(0)
+        else:
+            data = json.dumps({
+                "data": whole_station_response_json.get('data'),
+                "extra": {}
+            })
+            rds_list.push_whole_station_data_list(data)
+
+    except Exception as e:
+        print(
+            time.strftime("%H:%M:%S", time.localtime()) 
+            + ' ' 
+            + '数据异常：' 
+            + str(e)
+        )
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    print("主方法开始执行")
+
+    room_info = MysqlUserLiving().get_living_info()
+
+    if room_info is None:
+        sys.exit(0)
+    
+    room_id,anchor_id,sec_anchor_id = room_info
+    
+    task = threading.Thread(target=scrape, args=(room_id, sec_anchor_id, anchor_id,))
+    task.start()  # 准备就绪, 等待cpu执行

douyin_live_lottery_scraper.py

@@ -50,14 +50,32 @@ def get_mc():
         k += a() + ':'
     return k[:-1]
 
-def get_live_lottery_data(room_id):
+def get_trace():
+    trace_list = [
+        '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
+        '00-ce7faf4409b7fcc0ae6135fdd4250468-ce7faf4409b7fcc0-01',
+        '00-ce7faf3b09b7fcc0ae6042f1d8100468-ce7faf3b09b7fcc0-01',
+        '00-cdd79d2309b7fcc0ae6625a4cb190468-cdd79d2309b7fcc0-01',
+        '00-cefde9f009b7fcc0ae6750e1349e0468-cefde9f009b7fcc0-01',
+        '00-ced2e6ef09b7fcc0ae67dd7bfe000468-ced2e6ef09b7fcc0-01',
+        '00-cefbfeb509b7fcc0ae659396a6ea0468-cefbfeb509b7fcc0-01',
+        '00-cefaa25409b7fcc0ae657726a3c30468-cefaa25409b7fcc0-01',
+    ]
+
+    return random.choice(trace_list)
+
+def get_live_lottery_data(room_id, result):
     domain = 'webcast5-normal-ipv6-lf.amemv.com'
     url = 'https://' + domain + '/webcast/lottery/melon/check_user_right/?'
 
     rticket = str(int(time.time() * 1000))
     mc = get_mc
     udid = '8604' + get_random(11)
+    trace_id = get_trace()
     # openudid = '3b22' + str(udid.uuid4())[-12:]
+        
+
+    device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11]
     
     query = {
         "room_id" : room_id,
@@ -72,7 +90,7 @@ def get_live_lottery_data(room_id):
         "dpi" : "640",
         "app_name" : "aweme",
         "version_name" : "12.8.0",
-        "ts" : "1629688012",
+        "ts" : int(time.time()),
         "cpu_support64" : "true",
         "storage_type" : "0",
         "app_type" : "normal",
@@ -84,13 +102,13 @@ def get_live_lottery_data(room_id):
         "_rticket" : rticket,
         # "_rticket" : "1629688012123",
         "device_platform" : "android",
-        "iid" : "3158219122552359",
+        "iid" : iid,
         "version_code" : "120800",
         "mac_address" : mc,
         #"mac_address" : "FC%3ADB%3AB3%3A56%3ABD%3AFD",
         "cdid" : "6c96979e-c729-419c-9516-3a85a7338d0c",
-        "openudid" : "89bb178775bf2aa9",
-        "device_id" : "60314377681",
+        "openudid" : openudid,
+        "device_id" : device_id,
         "resolution" : "1440*2560",
         "os_version" : "6.0.1",
         "language" : "zh",
@@ -114,21 +132,21 @@ def get_live_lottery_data(room_id):
         'Cache-Control': 'max-age=0',
         'Upgrade-Insecure-Requests': '1',
         'User-Agent': userAgent,
-        'accept-encoding': 'gzip, deflate',
+        # 'accept-encoding': 'gzip, deflate',
         # "x-SS-REQ-TICKET": rticket,
         "x-gorgon": xGorgon.get('X-Gorgon'),
         "x-khronos": xGorgon.get('X-Khronos'),
         'passport-sdk-version' : '17',
         'sdk-version' : '2',
         'x-ss-dp' : '1128',
-        'x-tt-trace-id' : '00-70f99f2209e0b045dd14266ee1da0468-70f99f2209e0b045-01',
+        'x-tt-trace-id' : trace_id
     }
 
     retry = 0
     response_json = None
 
     while True:
-        if retry > 10:
+        if retry > 5:
             break
 
         retry += 1
@@ -151,6 +169,14 @@ def get_live_lottery_data(room_id):
                 response_json = response.json()
 
                 if (response_json.get('data') is not None):
+                    print(
+                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                        + ' 数据获取成功！' 
+                        + '\n'
+                        + room_id 
+                        + '\n'
+                    )
+
                     break
                 else:
                     print(
@@ -173,6 +199,8 @@ def get_live_lottery_data(room_id):
                     + '\n'
                     + room_id 
                     + '\n'
+                    + '爬取结果：' + str(response)
+                    + '\n'
                 )
                 time.sleep(1)
         except requests.exceptions.ProxyError as e:
@@ -230,8 +258,17 @@ def scrape():
         + str(room_id)
     )
 
+    key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221'
+    rdid = DbRedis.connect().rpop(key)
+    if rdid:
+        result = rdid.split('@@@')
+    else:
+        return None
+
+    DbRedis.connect().lpush(key, rdid)
+
     try:
-        response_json = get_live_lottery_data(room_id)
+        response_json = get_live_lottery_data(room_id, result)
        
         if response_json is None:
             # rds_list.record_score(0)
@@ -256,7 +293,7 @@ def scrape():
         rds_list.push_data_list(data)
     except Exception as e:
         # rds_list.record_score(0)
-        # rds_list.push_request_id(room_id)
+        rds_list.push_request_id(room_id)
         print(
             time.strftime("%H:%M:%S", time.localtime()) 
             + ' ' 

douyin_scraper_register.py

@@ -0,0 +1,52 @@
+import json
+from xlog03 import *
+from rds_model.db_redis import DbRedis
+from log.print_log import PrintLog
+import douyin_class
+
+redisModel = DbRedis.outer_net_connect()
+
+if __name__ == '__main__':
+    import warnings
+    warnings.filterwarnings("ignore")
+
+    print('===========注册程序===========')
+    PrintLog.print('===========注册程序===========')
+
+    rkey = 'DOUYIN_REGISTER_QUEUE'
+    cur_time = int(time.time())
+    while True:
+        now_time = int(time.time())
+        if (now_time - cur_time) > 270:
+            break
+
+        str_time = time.strftime("%H:%M:%S", time.localtime())
+        try:
+            llen = redisModel.llen(rkey)
+            if llen < 50:
+                proxies = {
+                    "http": "http://lum-customer-c_44eb09ec-zone-data_center:4jme8s5hf9x6@zproxy.lum-superproxy.io:22225",
+                    "https": "http://lum-customer-c_44eb09ec-zone-data_center:4jme8s5hf9x6@zproxy.lum-superproxy.io:22225"
+                }
+
+                print(str_time + ' 注册代理:' + str(proxies))
+                PrintLog.print(str_time + ' 注册代理:' + str(proxies))
+                douApi = douyin_class.DouYinApi('', proxies)
+
+                result = douApi.register_device()
+                if result['device_id'] == '0':
+                    continue
+
+                print(str_time + ' 结果:' + str(result))
+                PrintLog.print(str_time + ' 结果:' + str(result))
+
+                json_data = json.dumps(result)
+                redisModel.lpush(rkey, json_data)
+                print(str_time + ' 注册成功')
+                PrintLog.print(str_time + ' 注册成功')
+            else:
+                time.sleep(3)
+        except Exception as e:
+            print(str_time + ' 错误:' + str(e))
+            PrintLog.print(str_time + ' 错误:' + str(e))
+            continue

douyin_user_scraper_ab.py

@@ -0,0 +1,120 @@
+import json
+from xlog03 import *
+from rds_model.db_redis import DbRedis
+from log.print_log import PrintLog
+import douyin_class
+import random
+import threading
+
+redisModel = DbRedis.connect()
+redisDyModel = DbRedis.douyin_connect()
+
+def get_scraper_proxy(proxy_key):
+    proxy_dict = redisModel.hgetall(proxy_key)
+    if (proxy_dict is None) or (len(proxy_dict) == 0):
+        return
+    proxy_list = list(proxy_dict)
+
+    now = int(time.time())
+    while True:
+        proxy = random.choice(proxy_list)
+        if proxy is None:
+            return
+        proxy_info = proxy_dict.get(proxy)
+        if proxy_info is None:
+            continue
+        proxy_info = json.loads(proxy_info)
+        expire_at = int(proxy_info.get('expired_at'))
+        # 删除过期的代理
+        if expire_at <= now:
+            redisModel.hdel(proxy_key, proxy)
+            proxy_list.remove(proxy)
+            continue
+
+        return proxy
+
+
+def scraper():
+    rkey = 'DOUYIN_REGISTER_QUEUE'
+    pKey = 'IpProxyHash'
+    uKey = 'SL:List:Douyin:BarrageUserScrapeQueue'
+    sKey = 'SL:List:Douyin:BarrageUserDataQueue'
+
+    cur_time = int(time.time())
+    while True:
+        now_time = int(time.time())
+        if (now_time - cur_time) > 270:
+            print('thrend_' + threading.current_thread().name + ' finish')
+            break
+
+        rllen = redisDyModel.llen(rkey)
+        if rllen == 0:
+            time.sleep(2)
+            continue
+
+        ullen = redisDyModel.llen(uKey)
+        if ullen == 0:
+            time.sleep(2)
+            continue
+
+        json_data = redisDyModel.rpop(rkey)
+        str_time = time.strftime("%H:%M:%S", time.localtime())
+        try:
+            proxy = get_scraper_proxy(pKey)
+            if proxy:
+                proxies = {
+                    "http": "http://" + proxy,
+                    "https": "http://" + proxy
+                }
+            else:
+                time.sleep(2)
+                continue
+
+            print(str_time + ' 爬取代理:' + str(proxies))
+            PrintLog.print(str_time + ' 爬取代理:' + str(proxies))
+            douApi = douyin_class.DouYinApi('', proxies)
+
+            dict_data = json.loads(json_data)
+            device_id, iid, udid, openudid, cookie = dict_data['device_id'], dict_data['iid'], dict_data['uuid'], \
+                                                     dict_data['openudid'], dict_data['cookie']
+            douApi.init_device_ids(device_id, iid, udid, openudid)
+
+            user_id = redisDyModel.rpop(uKey)
+            response = douApi.get_user_info(user_id)
+            if len(response.text) > 0 and response.json()['status_code'] == 0 and response.json()['user']:
+                print(str_time + ' user_id:' + str(user_id) + ' 爬取成功')
+                PrintLog.print(str_time + ' user_id:' + str(user_id) + ' 爬取成功')
+                redisDyModel.lpush(sKey, response.text)
+                redisDyModel.lpush(rkey, json_data)
+            else:
+                print(str_time + ' user_id:' + str(user_id) + ' 爬取失败')
+                PrintLog.print(str_time + ' user_id:' + str(user_id) + ' 爬取失败')
+                scraper_time = dict_data['times']
+                if scraper_time < 10:
+                    dict_data['times'] += 1
+                    redisDyModel.lpush(rkey, json.dumps(dict_data))
+
+            time.sleep(1)
+        except Exception as e:
+            print(str_time + ' 错误:' + str(e))
+            PrintLog.print(str_time + ' 错误:' + str(e))
+            redisDyModel.lpush(rkey, json_data)
+            continue
+
+if __name__ == '__main__':
+    import warnings
+    warnings.filterwarnings("ignore")
+
+    print('===========爬取程序===========')
+    PrintLog.print('===========爬取程序===========')
+
+    threading_count = 10
+    for i in range(0, threading_count):
+        task = threading.Thread(target=scraper, name=i)
+        task.start()  # 准备就绪,等待cpu执行
+
+
+
+
+
+

douyin_user_scraper_in.py

@@ -0,0 +1,157 @@
+import requests
+from xlog03 import *
+from rds_model.db_redis import DbRedis
+import random
+import json
+import threading
+from log.print_log import PrintLog
+
+redisModel = DbRedis.connect()
+redisDyModel = DbRedis.douyin_connect()
+
+def get_user_info(user_id, proxies):
+    params = {
+        'user_id': user_id,
+        'version_code': '9.8.1',
+        'js_sdk_version': '1.47.2.2',
+        'app_name': 'aweme',
+        'app_version': '9.8.1',
+        'channel': 'App%20Store',
+        'mcc_mnc': 46002,
+        'aid': 1128,
+        'screen_width': 640,
+        'os_api': 18,
+        'ac': 'WIFI',
+        'os_version': '13.3.1',
+        'device_platform': 'iphone',
+        'build_number': 98107,
+        'device_type': 'iPhone8,4',
+        'address_book_access': 1
+    }
+    url = 'https://api3-normal-c-lf.amemv.com/aweme/v1/user/profile/self/?'
+
+    douyin_url = parse_params(url, params)
+    response = http_get(douyin_url, proxies)
+
+    return response
+
+def parse_params(url, params):
+    if params is None:
+        params = {}
+
+    if not url.endswith('?'):
+        url = url + '?'
+
+    common_params = parse.urlencode(params)
+    douyin_url = url + common_params
+
+    return douyin_url
+
+def http_get(douyin_url, proxies):
+    if proxies:
+        resp = requests.get(douyin_url, proxies=proxies, verify=False, timeout=10)
+    else:
+        resp = requests.get(douyin_url, verify=False, timeout=10)
+
+    return resp
+
+def get_scraper_proxy(proxy_key):
+    proxy_dict = redisModel.hgetall(proxy_key)
+    if (proxy_dict is None) or (len(proxy_dict) == 0):
+        return
+    proxy_list = list(proxy_dict)
+
+    now = int(time.time())
+    while True:
+        proxy = random.choice(proxy_list)
+        if proxy is None:
+            return
+        proxy_info = proxy_dict.get(proxy)
+        if proxy_info is None:
+            continue
+        proxy_info = json.loads(proxy_info)
+        expire_at = int(proxy_info.get('expired_at'))
+        # 删除过期的代理
+        if expire_at <= now:
+            redisModel.hdel(proxy_key, proxy)
+            proxy_list.remove(proxy)
+            continue
+
+        return proxy
+
+def scraper():
+    pKey = 'IpProxyHash'
+    uKey = 'SL:List:Douyin:BarrageUserScrapeQueue'
+    sKey = 'SL:List:Douyin:BarrageUserDataQueue'
+    hKey = 'SL:Hash:Douyin:BarrageUserScrapeRecord'
+
+    cur_time = int(time.time())
+    while True:
+        now_time = int(time.time())
+        if (now_time - cur_time) > 270:
+            print('thrend_' + threading.current_thread().name + ' finish')
+            break
+
+        ullen = redisDyModel.llen(uKey)
+        if ullen == 0:
+            time.sleep(2)
+            continue
+
+        str_time = time.strftime("%H:%M:%S", time.localtime())
+        user_id = redisDyModel.rpop(uKey)
+
+        try:
+            proxy = get_scraper_proxy(pKey)
+            if proxy:
+                proxies = {
+                    "http": "http://" + proxy,
+                    "https": "http://" + proxy
+                }
+            else:
+                time.sleep(2)
+                continue
+
+            print(str_time + ' 爬取代理:' + str(proxies))
+            PrintLog.print(str_time + ' 爬取代理:' + str(proxies))
+
+            response = get_user_info(user_id, proxies)
+            if len(response.text) > 0 and response.json()['status_code'] == 0 and response.json()['user']:
+                print(str_time + ' user_id:' + str(user_id) + ' 爬取成功')
+                PrintLog.print(str_time + ' user_id:' + str(user_id) + ' 爬取成功')
+                redisDyModel.lpush(sKey, response.text)
+                redisDyModel.hdel(hKey, user_id)
+            else:
+                print(str_time + ' user_id:' + str(user_id) + ' 爬取失败')
+                PrintLog.print(str_time + ' user_id:' + str(user_id) + ' 爬取失败')
+
+                record_json = redisDyModel.hget(hKey, user_id)
+                if record_json:
+                    record_dict = json.loads(record_json)
+                    if record_dict['times'] < 10:
+                        record_dict['times'] += 1
+                        redisDyModel.hset(hKey, user_id, json.dumps(record_dict))
+                    else:
+                        redisDyModel.hdel(hKey, user_id)
+                else:
+                    record_dict = {'times': 1}
+                    redisDyModel.hset(hKey, user_id, json.dumps(record_dict))
+
+            time.sleep(1)
+        except Exception as e:
+            print(str_time + ' 错误:' + str(e))
+            PrintLog.print(str_time + ' 错误:' + str(e))
+            redisDyModel.lpush(uKey, user_id)
+            continue
+
+if __name__ == '__main__':
+    import warnings
+    warnings.filterwarnings("ignore")
+
+    print('===========爬取程序===========')
+    PrintLog.print('===========爬取程序===========')
+
+    threading_count = 50
+    for i in range(0, threading_count):
+        task = threading.Thread(target=scraper, name=i)
+        task.start()  # 准备就绪,等待cpu执行
+

douyin_video_scraper_web.py

@@ -1,11 +1,15 @@
-import json,time,sys,threading,warnings
+import json
+import sys
+import threading
+import time
+import warnings
 
-from rds_model.db_redis import DbRedis
-from log.print_log import PrintLog
 from libs.proxy import Proxy
-from web_dy import *
+from log.print_log import PrintLog
+from rds_model.db_redis import DbRedis
 from rds_model.rds_user_video_list import RdsUserVideoList
 from web_cookie import Cookie
+from web_dy import *
 
 start_time = time.time()
 # -- coding: utf-8 --**
@@ -56,10 +60,25 @@ def get_signature(url=None,method='_signature'):
     
     return d
 
+def get_ua_ck():
+    ua_list=[
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400"
+    ]
+    ck_list=[
+        'ttwid=1%7CTVzdM0P0u-8dtsmh6c-EaQEtBoTSOs_MG85FAg07AbA%7C1631502013%7C66442d8594de8e93ad18b73f3dfe0c94ed864c3d932824bcde9918b5be172321; passport_csrf_token=866923f1a32045fd82e47053158402a2',
+        'ttwid=1%7CGPDDu9-w3RGs2Pcd0wRlvLYoktpDt-v8LP5ZMyb1NBM%7C1630319594%7Cffb8de47e6da87dcfd76349b5ad34aa1f9b9d4332261a3a8436b932a893366c1; passport_csrf_token=79284b8777a7a54f3066cefef9af539e',
+        'ttwid=1%7CGsfqc7NpdOg4N-U-VX7Q77KsWjVTZ7gxLNifsisj8YE%7C1631618570%7Cafbb13a27fd1c2d7a8245454b1e0d7cd654d80848a320933a25d9ef77638c18c; passport_csrf_token=84911c9af94040a99cc10416bd27533d',
+        'ttwid=1%7C82FGr05YUOReYUB301ao_erqOQ3ilbXZdEy0tkMsdXY%7C1631863641%7C1dcebe643a96f00841a3b490db60de886bfe07ff3d276e509717abc4e1681ba6; passport_csrf_token=494ae3fffe00328101fd40e050ce49db',
+        'ttwid=1%7CwfnX3T9LY4_60iGoQNzyqYe5ahILFeRxfMuZ1pdgXf8%7C1632724192%7Cb613fddc0b533d5578dad4d5f9290705fdc6432aa854d492f4761d164dd3fdd5; passport_csrf_token=4a8afba333103b033e537003b72ee91b'
+    ]
+    return random.choice(ua_list),random.choice(ck_list)
+
 def get_user_videos(sec_user_id, max_cursor=0, count=20):
 
-    # ua,ck=get_ua_ck('get_user_videos')
-    ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    ua,ck=get_ua_ck()
+    # ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
     
     url='https://www.douyin.com/aweme/v1/web/aweme/post/?'
     param={
@@ -70,8 +89,8 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
         'max_cursor': str(max_cursor),
         'count': str(count),
         'publish_video_strategy_type': '2',
-        'version_code': '160100',
-        'version_name': '16.1.0',
+        'version_code': '170400',
+        'version_name': '17.4.0',
         'cookie_enabled': 'true',
         'screen_width': '1920',
         'screen_height': '1080',
@@ -80,6 +99,7 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
         'browser_name': 'Mozilla',
         'browser_version':ua.replace('Mozilla/',''),
         'browser_online': 'true',
+        "source" : "channel_pc_web"
     }
 
     url = url + parse.urlencode(param)
@@ -104,6 +124,12 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
         "referer": "https://www.douyin.com/user/{sec_user_id}?enter_method=search_result&enter_from=search_result".format(sec_user_id=sec_user_id),
         "user-agent":ua,
         "withcredentials": "true",
+        "sec-ch-ua" : '"Google Chrome";v="93", " Not;A Brand";v="99", "Chromium";v="93"',
+        "sec-ch-ua-mobile" : "?0",
+        "sec-ch-ua-platform" : "Windows",
+        "sec-fetch-dest" : "empty",
+        "sec-fetch-mode" : "cors",
+        "sec-fetch-site" : "same-origin"
     }
 
     if ck:
@@ -134,6 +160,7 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
                 timeout=8
             )
             if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
+                # print(response)
                 response_json = response.json()
 
                 if (response_json.get('aweme_list') is not None):
@@ -164,6 +191,8 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
                     + '\n'
                     + str(sec_user_id) 
                     + '\n'
+                    + '爬取结果：' + str(response)
+                    + '\n'
                 )
                 time.sleep(1)
         except requests.exceptions.ProxyError as e:
@@ -196,12 +225,14 @@ def get_user_videos(sec_user_id, max_cursor=0, count=20):
                 + ' 请求抛出异常！' 
                 + str(e) 
                 + '\n'
+                + str(e.__traceback__.tb_lineno)
+                + '\n'
                 + str(sec_user_id) 
                 + '\n'
                 + Proxy.proxy_info
             )
             pass
-
+        
     return response_json
 
 def scrape():
@@ -221,7 +252,7 @@ def scrape():
     )
 
     try:
-        videos = get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=20)
+        videos = get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=50)
        
         if videos is None:
             # rds.push_request_id(sec_user_id)
@@ -299,4 +330,4 @@ if __name__ == '__main__':
             )
             sys.exit(0)
 
-        time.sleep(1)
+        time.sleep(1)

dy_barrage_scraper.py

@@ -48,7 +48,7 @@ def scrape():
                 if int(status) == 4:     # 直播已结束不再回塞数据到爬取队列
                     continue
 
-            # 直播未结束塞回队列
+            # 直播未结束爬取完成塞回队列
             time_diff = int(time.time()) - int(scrape_time)
             if time_diff > 5:
                 # 爬取前更新爬取时间塞回队列

dy_live_reward.py

@@ -0,0 +1,100 @@
+import time
+import threading
+import json
+import sys
+
+from rds_model.rds_dy_live_reward_request_list import RdsDyLiveRewardRequestList
+from log.print_log import PrintLog
+from libs.mysql_dy_live import MysqlDyLive
+from libs.dy_live_reward_info import DyLiveRewardInfo
+
+
+def scrape():
+    while True:
+        try:
+            # 从 待爬取直播间 列表中获取一个 直播间信息
+            room_info = rds.get_request_param()
+
+            # 如果没有待爬取的直播，则等一秒，循环
+            if room_info is None:
+                time.sleep(1)
+                continue
+
+            # 判断是否到达爬取时间以确定是否需要爬取弹幕，并直接塞回队列尾部
+            room_dict = json.loads(room_info)
+            room_id = str(room_dict['room_id'])  # 直播间ID
+            scrape_time = room_dict['scrape_time']  # 上次抓取时间
+            uid = room_dict['uid']      # 直播网红ID
+
+            if (uid is None) or (room_id is None):
+                PrintLog.print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求数据数据异常！' + '\n'
+                    + room_info
+                )
+                time.sleep(1)
+                continue
+
+            room_dict.setdefault('times', 0)
+            room_dict['times'] = (room_dict['times'] + 1) % 10
+
+            # 按照策略间隔性判断直播是否已停播
+            if room_dict['times'] == 0:  # 每爬取十次判断一次是否已关播
+                # 判断直播是否结束
+                live_info = MysqlDyLive().get_live_info(room_id)
+                if live_info is None:  # 直播不存在，不再回塞数据到爬取队列
+                    continue
+
+                pk_id, room_id, status = live_info
+
+                if int(status) == 4:  # 直播已结束不再回塞数据到爬取队列
+                    continue
+
+            time_diff = int(time.time()) - int(scrape_time)
+
+            if time_diff > 120:
+                # 爬取前更新爬取时间塞回队列
+                room_dict['scrape_time'] = int(time.time())
+                rds.push_request_id(json.dumps(room_dict))
+
+                response_json = DyLiveRewardInfo.get_data(room_id)
+                if (response_json.get('data') is None) or (response_json.get('data').get('ranks') is None):
+                    PrintLog.print(time.strftime("%H:%M:%S", time.localtime()) + ' 获取打赏数据异常：' + room_id + ' ' + uid)
+                    rds.record_score(0)
+                    continue
+
+                # 没有商品
+                if len(response_json.get('data').get('ranks')) == 0:
+                    PrintLog.print(time.strftime("%H:%M:%S", time.localtime()) + ' 没有打赏数据：' + room_id + ' ' + uid)
+                    continue
+
+                data = json.dumps({
+                    "data": response_json.get('data'),
+                    "extra": {
+                        'room_id': room_id,
+                        'uid': uid,
+                    }
+                })
+
+                rds.record_score(1)
+                rds.push_data_list(data)
+            else:
+                print('直播ID%s %d秒前曾爬取过，暂无需继续抓取' % (room_id, time_diff))
+                time.sleep(1)
+                rds.push_request_id(json.dumps(room_dict))
+
+        except Exception as e:
+            PrintLog.print(time.strftime("%H:%M:%S", time.localtime()) + ' ' + room_id + '数据异常：' + str(e))
+
+        time.sleep(0.1)
+
+
+if __name__ == "__main__":
+    print("主方法开始执行")
+
+    rds = RdsDyLiveRewardRequestList()
+
+    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' ' + ' 开始执行，待爬取直播队列长度：' + str(rds.get_len()))
+
+    for i in range(1, 50):
+        task = threading.Thread(target=scrape, name=i)
+        task.start()  # 准备就绪,等待cpu执行

libs/dy_live_reward_info.py

@@ -0,0 +1,79 @@
+import requests
+import time
+
+from libs.proxy import Proxy
+from log.print_log import PrintLog
+from libs.en0414 import DouYinApi
+
+
+class DyLiveRewardInfo:
+    @staticmethod
+    def get_data(room_id, retry=0):
+        while True:
+            if retry > 10:
+                break
+
+            retry += 1
+
+            proxy = Proxy.get()
+
+            proxies = {
+                "http": "http://" + proxy,
+                "https": "http://" + proxy
+            }
+
+            try:
+                dou_api = DouYinApi('', proxies)
+                result = dou_api.register_device()
+                device_id, iid, udid, openudid, cookie = result['device_id'], result['iid'], result['uuid'], result[
+                    'openudid'], result['cookie']
+                dou_api.init_device_ids(device_id, iid, udid, openudid)
+
+                params = {
+                    "has_market": "0",
+                    "is_activated": '0'
+                }
+                dou_api.comm_get('https://aweme.snssdk.com/service/2/app_alert/?', params)
+
+                response_json = dou_api.get_webcast_ranklist(room_id)
+
+                if (response_json is not None) and (response_json.get('status_code') == 0):
+                    print('成功：' + room_id)
+                    PrintLog.print('成功：' + room_id)
+                    return response_json
+
+                if retry >= 10:
+                    print('失败：' + room_id + '***')
+                    PrintLog.print('失败：' + '***')
+                    return None
+
+                retry += 1
+
+                return DyLiveRewardInfo.get_data(room_id, retry)
+            except requests.exceptions.ProxyError as e:
+                PrintLog.print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '代理过期！' + str(e) + '\n'
+                    + room_id + '\n'
+                    + Proxy.proxy_info
+                )
+                Proxy.del_proxy(proxy)
+                pass
+            except requests.exceptions.ConnectTimeout as e:
+                PrintLog.print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' ConnectTimeout！' + str(e) + '\n'
+                    + room_id + '\n'
+                    + Proxy.proxy_info
+                )
+                Proxy.del_proxy(proxy)
+                pass
+            except Exception as e:
+                PrintLog.print(
+                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求抛出异常！' + str(e) + '\n'
+                    + room_id
+                )
+                pass
+
+
+if __name__ == "__main__":
+    room_id = '6955048598123891496'
+    commodity_detail = DyLiveRewardInfo.get_data(room_id)

libs/en0414.py

@@ -934,7 +934,7 @@ class DouYinApi:
             a = parse.urlencode(form_params)
             stub = hashlib.md5(a.encode('utf-8')).hexdigest()
         ts = int(time.time())
-        from lib_scraper.Xg04 import X_Gorgon
+        from libs.Xg04 import X_Gorgon
         params = url[url.index('?') + 1:]
         result = X_Gorgon(params, "", cookie)
         gorgon = result['X-Gorgon']

libs/mysql_user_living.py

@@ -6,7 +6,7 @@ class MysqlUserLiving(DbMysql):
         super().__test__()
 
     def get_living_info(self):
-        sql = 'SELECT `room_id`,l.`uid`,u.`sec_uid` FROM douyin_live as l left join douyin_users as u on l.`uid`=u.`uid` WHERE `finish_time`=0 order by u.`follower_count` desc LIMIT 1'
+        sql = 'SELECT `room_id`,l.`uid`,u.`sec_uid` FROM douyin_live as l left join douyin_users as u on l.`uid`=u.`uid` WHERE `finish_time`=0 and `has_commerce_goods`=1 order by l.`sell_count_sum` desc LIMIT 1'
         try:
             # 查询数据
             self.test.execute(sql)

rds_model/db_redis.py

@@ -36,4 +36,20 @@ class DbRedis:
                 decode_responses=True
             )
 
-        return DbRedis.__douyin_connection
+        return DbRedis.__douyin_connection
+
+    @staticmethod
+    def outer_net_connect():
+        if DbRedis.__connection is None:
+            host = 'r-2ze28bdb7389a8a4pd.redis.rds.aliyuncs.com'
+            port = 6379
+            password = 'Zhuaduoduo2017'
+
+            DbRedis.__connection = redis.StrictRedis(
+                host=host,
+                port=port,
+                password=password,
+                decode_responses=True
+            )
+
+        return DbRedis.__connection

rds_model/rds_dy_live_reward_request_list.py

@@ -0,0 +1,30 @@
+from rds_model.db_redis import DbRedis
+
+
+class RdsDyLiveRewardRequestList:
+    def __init__(self):
+        self.redis = DbRedis.connect()
+
+    # 获取待爬取直播打赏的队列长度
+    def get_len(self):
+        key = 'BrandLiveData.DyUserRewardScraperWaiting'
+        return self.redis.llen(key)
+
+    # 获取待爬取打赏数据的直播间信息
+    def get_request_param(self):
+        key = 'BrandLiveData.DyUserRewardScraperWaiting'
+        return self.redis.rpop(key)
+
+    # 塞回队列
+    def push_request_id(self, data):
+        key = 'BrandLiveData.DyUserRewardScraperWaiting'
+        return self.redis.lpush(key, data)
+
+    # 打赏数据存储至队列等待处理
+    def push_data_list(self, data):
+        key = 'BrandLiveData.DyLiveRewardsDataList'
+        return self.redis.lpush(key, data)
+
+    def record_score(self, data):
+        key = 'BrandLiveData.DyLiveRewardsScore'
+        self.redis.lpush(key, data)

web_cookie.py

@@ -47,7 +47,7 @@ class Cookie:
             create_at = int(time.mktime(time.strptime(cookie_info, "%Y-%m-%d %H:%M:%S")))
 
             # 删除过期的代理
-            if now - create_at >= 3600:
+            if now - create_at >= 7200:
                 Cookie.redis.hdel(key, cookie)
                 cookie_list.remove(cookie)
                 continue
@@ -64,7 +64,7 @@ class Cookie:
 
         now = int(time.time())
 
-        if create_at - now >= 3600:
+        if create_at - now >= 7200:
             return
 
         # 删除失效的Cookie
@@ -73,7 +73,23 @@ class Cookie:
     def set(cookie):
         Cookie.redis.hset('CookieHash', cookie, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
 
+    def get_url(self):
+        url_list = [
+            "https://www.douyin.com/user/MS4wLjABAAAAKpCGhwidAtgmUXmYIT0zjp2QpGquUaOCEeVPE6_gHjQ",
+            "https://www.douyin.com/user/MS4wLjABAAAA3Gq3QsbpkmIchOTXXF51Fy3Dyb0xF7rLvI3QEQjoYwo",
+            "https://www.douyin.com/user/MS4wLjABAAAAO6GOEmNyHKo8Kd1IdS95J884BAFTpR8eRDOBIAb2d7VNGWFBzhR2odpulkKJFV95",
+            "https://www.douyin.com/user/MS4wLjABAAAAc6xH1Jxur09z-Oy5M9IVpckmAyPlQg5uj_B8fFrTIiNv5B-XyH6G9RCQt3qLsZZU",
+            "https://www.douyin.com/user/MS4wLjABAAAA5v-bl3BibVonLiHoZWW173nKKy_yn2DACHErPbOoo3g",
+            "https://www.douyin.com/user/MS4wLjABAAAATjfBnSaWkl6ZJiznrCzkt7-l_7pkD4pLkKyPkLtD2VU",
+            "https://www.douyin.com/user/MS4wLjABAAAAZ-hqjDujmsKhlhuNV1R4OPrIWJ0XmhFnuJmy9h3u4VkovGqJycoOVGaCj8uqwQiJ",
+            "https://www.douyin.com/user/MS4wLjABAAAAoMNLc-_Vx_TDrJQvpGEtgZfpR99JDKf6n23mXpsnMGg",
+            "https://www.douyin.com/user/MS4wLjABAAAAMhl868Pj7GIBYNVX46kjLGS_eiprGHaDHe5ffqi91_s",
+            "https://www.douyin.com/user/MS4wLjABAAAAR8ow3aH-TjB2c4TJFqtDvFPhmd3TBFyHF1zMCLd39rdO45zpMZXAIvdwsQ4_7gw6"
+        ]
+        return random.choice(url_list)
+
     def get_ck(self):
+
         requests.packages.urllib3.disable_warnings()
         headers = {
             'authority': 'www.douyin.com',
@@ -92,6 +108,8 @@ class Cookie:
         # 屏蔽验证https证书提示
         requests.packages.urllib3.disable_warnings()
 
+        url = self.get_url()
+
         while True:
             if retry > 10:
                 break
@@ -107,7 +125,7 @@ class Cookie:
 
             try:
                 response = requests.get(
-                    'https://www.douyin.com/',
+                    url,
                     headers=headers,
                     proxies=proxies,
                     timeout=8,
@@ -123,17 +141,29 @@ class Cookie:
                     ck=ck[:-2]
                     ck+='; passport_csrf_token='+str(uuid.uuid4()).replace('-','')
 
-                    Cookie.set(ck)
-                    
-                    print(
-                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
-                        + ' 数据获取成功！' 
-                        + '\n'
-                        + ck
-                        + '\n'
-                        + Proxy.proxy_info
-                    )
-                    break
+                    num = ck.count('ttwid')
+                    if num>0:
+                        Cookie.set(ck)
+                        print(
+                            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                            + ' 数据获取成功！' 
+                            + '\n'
+                            + ck
+                            + '\n'
+                            + Proxy.proxy_info
+                        )
+                        break
+                    else:
+                        print(
+                            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
+                            + ' 数据获取失败！' 
+                            + '\n'
+                            + ck
+                            + '\n'
+                            + Proxy.proxy_info
+                            + str(cookie_dict)
+                        )
+                        break
                 else:
                     print(
                         time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 

web_dy.py

@@ -1,7 +1,7 @@
-import requests,json,random,execjs,uuid,os
+import requests,json,random,execjs,uuid
 from urllib import parse
 from urllib.parse import quote
-from libs.proxy import Proxy
+# from libs.proxy import Proxy
 # -- coding: utf-8 --**
 
 class WebDouYin:
@@ -9,14 +9,14 @@ class WebDouYin:
         self.proxies = {
         }
     def get_signature(self,url=None,method='_signature'):
-
-        with open(os.getcwd()+"/signature.js", 'r', encoding='utf-8') as f:
+        with open('signature.js', 'r', encoding='utf-8') as f:
             b = f.read()
         
         c = execjs.compile(b)
         
         # url=url.replace('%28','(').replace('%29',')').replace('%2C',',')
         d = c.call(method, url.replace('\n',''))
+        
         # print('_signature',d)
         return d
     def get_ck(self, proxy=None):
@@ -32,7 +32,9 @@ class WebDouYin:
             'upgrade-insecure-requests': '1',
             'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
         }
-        res = requests.get('https://www.douyin.com/', headers=headers,verify=False, proxies=proxy,timeout=8)
+        # res = requests.get('https://www.douyin.com/', headers=headers,verify=False, proxies=proxy,timeout=8)
+        res = requests.get("https://www.douyin.com/user/MS4wLjABAAAAKpCGhwidAtgmUXmYIT0zjp2QpGquUaOCEeVPE6_gHjQ", headers=headers,verify=False, proxies=proxy,timeout=8)
+
         cookie_dict=res.cookies.get_dict()
         ck=''
 
@@ -48,11 +50,15 @@ class WebDouYin:
     def get_ua_ck(self,type_name=None):
         ua_list=[
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
+            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400"
         ]
         ck_list=[
-            # 'ttwid=1%7CTVzdM0P0u-8dtsmh6c-EaQEtBoTSOs_MG85FAg07AbA%7C1631502013%7C66442d8594de8e93ad18b73f3dfe0c94ed864c3d932824bcde9918b5be172321; passport_csrf_token=866923f1a32045fd82e47053158402a2',
-            # 'ttwid=1%7CGPDDu9-w3RGs2Pcd0wRlvLYoktpDt-v8LP5ZMyb1NBM%7C1630319594%7Cffb8de47e6da87dcfd76349b5ad34aa1f9b9d4332261a3a8436b932a893366c1; passport_csrf_token=79284b8777a7a54f3066cefef9af539e'
-            'ttwid=1%7CGsfqc7NpdOg4N-U-VX7Q77KsWjVTZ7gxLNifsisj8YE%7C1631618570%7Cafbb13a27fd1c2d7a8245454b1e0d7cd654d80848a320933a25d9ef77638c18c; passport_csrf_token=84911c9af94040a99cc10416bd27533d'
+            'ttwid=1%7CTVzdM0P0u-8dtsmh6c-EaQEtBoTSOs_MG85FAg07AbA%7C1631502013%7C66442d8594de8e93ad18b73f3dfe0c94ed864c3d932824bcde9918b5be172321; passport_csrf_token=866923f1a32045fd82e47053158402a2',
+            'ttwid=1%7CGPDDu9-w3RGs2Pcd0wRlvLYoktpDt-v8LP5ZMyb1NBM%7C1630319594%7Cffb8de47e6da87dcfd76349b5ad34aa1f9b9d4332261a3a8436b932a893366c1; passport_csrf_token=79284b8777a7a54f3066cefef9af539e',
+            'ttwid=1%7CGsfqc7NpdOg4N-U-VX7Q77KsWjVTZ7gxLNifsisj8YE%7C1631618570%7Cafbb13a27fd1c2d7a8245454b1e0d7cd654d80848a320933a25d9ef77638c18c; passport_csrf_token=84911c9af94040a99cc10416bd27533d',
+            'ttwid=1%7C82FGr05YUOReYUB301ao_erqOQ3ilbXZdEy0tkMsdXY%7C1631863641%7C1dcebe643a96f00841a3b490db60de886bfe07ff3d276e509717abc4e1681ba6; passport_csrf_token=494ae3fffe00328101fd40e050ce49db',
+            'ttwid=1%7CwfnX3T9LY4_60iGoQNzyqYe5ahILFeRxfMuZ1pdgXf8%7C1632724192%7Cb613fddc0b533d5578dad4d5f9290705fdc6432aa854d492f4761d164dd3fdd5; passport_csrf_token=4a8afba333103b033e537003b72ee91b'
         ]
         return random.choice(ua_list),random.choice(ck_list)
     def response(self,url,headers,proxy,data=None):
@@ -144,7 +150,6 @@ class WebDouYin:
             'browser_version':ua.replace('Mozilla/',''),
             'browser_online': 'true',
         }
-
         url = url + parse.urlencode(param)
         _signature = self.get_signature(url)
         url+='&_signature='+quote(_signature)
@@ -161,21 +166,21 @@ class WebDouYin:
             "user-agent":ua,
             "withcredentials": "true",
         }
-
         if ck:headers['cookie']=ck
         json_data=self.response(url=url,headers=headers,proxy=proxy)
         return json_data
 if __name__ == '__main__':
     webdy=WebDouYin()
-    # res = webdy.get_user_videos('MS4wLjABAAAAqLPgx-hHf27EqGEtRQ6YyuQQTmikB5CBO1jXy61yhWKujGd8KO5G8V2vdcLQJAym')
-    # print(res)
-    info = webdy.get_user_info('MS4wLjABAAAAHYNHFpUR36AQSxdDpSFrI2uM4aDvSF-8vjtjNiLepD0')
-    proxy = Proxy.get()
-    print(proxy)
-    proxies = {
-        "http": "http://" + proxy,
-        "https": "http://" + proxy
-    }
+    res = webdy.get_user_videos('MS4wLjABAAAAqLPgx-hHf27EqGEtRQ6YyuQQTmikB5CBO1jXy61yhWKujGd8KO5G8V2vdcLQJAym')
+    print(res)
+    # webdy.get_user_info('MS4wLjABAAAAC2euvL-0qMZyd80aNwZa-wX5KXuz_r7YVNHSBOogfVg')
+
+    # proxy = Proxy.get()
+    # print(proxy)
+    # proxies = {
+    #     "http": "http://" + proxy,
+    #     "https": "http://" + proxy
+    # }
 
-#    ck = webdy.get_user_info("MS4wLjABAAAAC2euvL-0qMZyd80aNwZa-wX5KXuz_r7YVNHSBOogfVg")
-#    print(ck)
+    # ck = webdy.get_ck()
+    # print(ck)