shop_live_scraper/web_dy.py

import os

import requests,json,random,execjs,uuid
from urllib import parse
from urllib.parse import quote
# from libs.proxy import Proxy
# -- coding: utf-8 --**

class WebDouYin:
    def __init__(self):
        self.proxies = {
        }
    def get_signature(self,url=None,method='_signature'):
        dirname, filename = os.path.split(os.path.abspath(__file__))
        with open(dirname+ '/signature.js', 'r', encoding='utf-8') as f:
            b = f.read()
        
        c = execjs.compile(b)
        
        # url=url.replace('%28','(').replace('%29',')').replace('%2C',',')
        d = c.call(method, url.replace('\n',''))
        
        # print('_signature',d)
        return d
    def get_ck(self, proxy=None):
        requests.packages.urllib3.disable_warnings()
        headers = {
            'authority': 'www.douyin.com',
            'method': 'GET',
            'path': '/',
            'scheme': 'https',
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
        }
        # res = requests.get('https://www.douyin.com/', headers=headers,verify=False, proxies=proxy,timeout=8)
        res = requests.get("https://www.douyin.com/user/MS4wLjABAAAAKpCGhwidAtgmUXmYIT0zjp2QpGquUaOCEeVPE6_gHjQ", headers=headers,verify=False, proxies=proxy,timeout=8)

        cookie_dict=res.cookies.get_dict()
        ck=''

        if cookie_dict:
            for k,v in cookie_dict.items():
                ck+='%s=%s; '%(k,v)
            ck=ck[:-2]
        else:
            return None
        ck+='; passport_csrf_token='+str(uuid.uuid4()).replace('-','')
        # print(ck)
        return ck

    def get_ua_ck(self,type_name=None):
        ua_list=[
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400"
        ]
        ck_list=[
            'ttwid=1%7CTVzdM0P0u-8dtsmh6c-EaQEtBoTSOs_MG85FAg07AbA%7C1631502013%7C66442d8594de8e93ad18b73f3dfe0c94ed864c3d932824bcde9918b5be172321; passport_csrf_token=866923f1a32045fd82e47053158402a2',
            'ttwid=1%7CGPDDu9-w3RGs2Pcd0wRlvLYoktpDt-v8LP5ZMyb1NBM%7C1630319594%7Cffb8de47e6da87dcfd76349b5ad34aa1f9b9d4332261a3a8436b932a893366c1; passport_csrf_token=79284b8777a7a54f3066cefef9af539e',
            'ttwid=1%7CGsfqc7NpdOg4N-U-VX7Q77KsWjVTZ7gxLNifsisj8YE%7C1631618570%7Cafbb13a27fd1c2d7a8245454b1e0d7cd654d80848a320933a25d9ef77638c18c; passport_csrf_token=84911c9af94040a99cc10416bd27533d',
            'ttwid=1%7C82FGr05YUOReYUB301ao_erqOQ3ilbXZdEy0tkMsdXY%7C1631863641%7C1dcebe643a96f00841a3b490db60de886bfe07ff3d276e509717abc4e1681ba6; passport_csrf_token=494ae3fffe00328101fd40e050ce49db',
            'ttwid=1%7CwfnX3T9LY4_60iGoQNzyqYe5ahILFeRxfMuZ1pdgXf8%7C1632724192%7Cb613fddc0b533d5578dad4d5f9290705fdc6432aa854d492f4761d164dd3fdd5; passport_csrf_token=4a8afba333103b033e537003b72ee91b',
            'ttwid=1%7CJZaclhj5Tgq6GB2KqL_6iqSu-BXCe9nC6-nJcjMEq24%7C1631262845%7Caf3e3acea147096036503a0238eccf3f30bbf31ac39d1812d69031aea79d3760; passport_csrf_token_default=b5e15d90e956f6b148bd3301ce6372f3; passport_csrf_token=b5e15d90e956f6b148bd3301ce6372f3; _tea_utm_cache_6383=undefined; __ac_signature=_02B4Z6wo00f01rm3RmAAAIDBLWEdbNgZ9Vq5k0LAAM9GpUBscdxSeV9NlDZq3ATFdnICqNTCV8cWCBBshe4Q9933w4I3vFIC2-ptR-6iU1HM7ufqE3zM2UJOe-WBklhX6bsTmlQaXggHi-rU16; FOLLOW_YELLOW_POINT_USER=MS4wLjABAAAAwcPuAI04fw8wfirPJqOv0hESAX-C0JbpI8JGzlgiX4s; FOLLOW_YELLOW_POINT_STATUE_INFO=1%2F1632903705967; tt_scid=ebxRLAmEaT7aTKjBzBzpqgKSCESII30busIfS0zONRZ6RFQ5Gmc52w58czEt3VQwd79e; s_v_web_id=verify_c5299c865f78fab640e630fad1f812cc; _tea_utm_cache_2018=undefined; odin_tt=c68622b8715a3aa3bfe289438613de2ffc711f9ceff12da4274f374e0c55268dce2f26fff0e7e3f1e0c645fe6491ee932a7e4b52a9be128e15ff8d49ace7822c; n_mh=1eNiy1CE3tbbhhZrqHZ8hb46OcXZmYiO3OSMVDwg0OE; sso_auth_status=aca31d09b431af5fb60642f0869a9112; sso_auth_status_ss=aca31d09b431af5fb60642f0869a9112; sso_uid_tt=d03690992f052ea087d48027b943a927; sso_uid_tt_ss=d03690992f052ea087d48027b943a927; toutiao_sso_user=cb6f6e70b5384aa94013e6cf5decb883; toutiao_sso_user_ss=cb6f6e70b5384aa94013e6cf5decb883; d_ticket=80ed5c329ed10dc1bb91b58dbecd3ac5bf128; passport_auth_status_ss=3be526de4218e101e6b5136365c332f8%2C18b527f1a55ff3bb63d05496e6ee56f5; sid_guard=b69c6f12202d7069d80061d32981f573%7C1632902665%7C5184000%7CSun%2C+28-Nov-2021+08%3A04%3A25+GMT; uid_tt=b536b77fe52631203eb01326194a96a1; uid_tt_ss=b536b77fe52631203eb01326194a96a1; sid_tt=b69c6f12202d7069d80061d32981f573; sessionid=b69c6f12202d7069d80061d32981f573; sessionid_ss=b69c6f12202d7069d80061d32981f573; sid_ucp_v1=1.0.0-KDU1YTg0M2Q5NTNhNmM2ZDI2NTVmM2Q5Y2E3NTUyM2E2ZTM4YzRhZWMKFQj1za_V-AIQibzQigYY7zE4AkDxBxoCaGwiIGI2OWM2ZjEyMjAyZDcwNjlkODAwNjFkMzI5ODFmNTcz; ssid_ucp_v1=1.0.0-KDU1YTg0M2Q5NTNhNmM2ZDI2NTVmM2Q5Y2E3NTUyM2E2ZTM4YzRhZWMKFQj1za_V-AIQibzQigYY7zE4AkDxBxoCaGwiIGI2OWM2ZjEyMjAyZDcwNjlkODAwNjFkMzI5ODFmNTcz; passport_auth_status=3be526de4218e101e6b5136365c332f8%2C18b527f1a55ff3bb63d05496e6ee56f5; msToken=Uk1zRW_2ZV5xpYN_5QTApoG2-zrYuL-qb4Y3bnuZRwm5LCauYqzp8sNlXMRlqDmwxpirVVBtovX0vano-KRPs_Xnssp6JayMGeWrxzmeV3Kw_jeD2dCZZah_0dRt; msToken=LH6B0ufdfs8Z66KLcA_ox0ZUoEnlgydp9-o2nelv2KxYzoWS6gB3QPLuJYIpcXhK0A6161RECrSy6-mECAz0GgCPrRtU-BdRzeILTOwDwu7FqV4S9p6OyJAA9fri'
        ]
        return random.choice(ua_list),random.choice(ck_list)
    def response(self,url,headers,proxy,data=None):
        # -- coding: utf-8 --**
        try:
            requests.packages.urllib3.disable_warnings()
            if data:
                res = requests.post(url, headers=headers,verify=False, proxies=proxy,data=data, timeout=8)
            else:
                res=requests.get(url,headers=headers,verify=False,proxies=proxy,timeout=8)
            # print(res)
            json_data=json.loads(res.content.decode())
        except Exception as e:
            json_data = "请求失败" + str(e)
            print(json_data)
        return json_data
    def get_user_videos(self,sec_user_id,max_cursor=0,count=10,proxy=None,cookie=None):
        
        ua,ck=self.get_ua_ck('get_user_videos')

        if cookie:
            ck = cookie
        
        url='https://www.douyin.com/aweme/v1/web/aweme/post/?'
        param={
            'device_platform': 'webapp',
            'aid': '6383',
            'channel': 'channel_pc_web',
            'sec_user_id': sec_user_id,
            'max_cursor': str(max_cursor),
            'count': str(count),
            'publish_video_strategy_type': '2',
            'version_code': '160100',
            'version_name': '16.1.0',
            'cookie_enabled': 'true',
            'screen_width': '1920',
            'screen_height': '1080',
            'browser_language': 'zh-CN',
            'browser_platform': 'Win32',
            'browser_name': 'Mozilla',
            'browser_version':ua.replace('Mozilla/',''),
            'browser_online': 'true',
        }

        url = url + parse.urlencode(param)
        
        _signature = self.get_signature(url)
        url+='&_signature='+quote(_signature)
        
        headers = {
            "authority": "www.douyin.com",
            "method": "GET",
            "path": str(url).replace('https://www.douyin.com',''),
            "scheme": "https",
            "accept": "application/json, text/plain, */*",
            # "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cookie": ck,
            "referer": "https://www.douyin.com/user/{sec_user_id}?enter_method=search_result&enter_from=search_result".format(sec_user_id=sec_user_id),
            "user-agent":ua,
            "withcredentials": "true",
        }
        if ck:headers['cookie']=ck
        json_data=self.response(url=url,headers=headers,proxy=proxy)
        
        return json_data
        # print(json_data)
    def get_user_info(self,sec_user_id,proxy=None):
        ua,ck=self.get_ua_ck()

        # 随机获取一个cookie

        url='https://www.douyin.com/aweme/v1/web/user/profile/other/?'
        param={
            'device_platform': 'webapp',
            'aid': '6383',
            'channel': 'channel_pc_web',
            'publish_video_strategy_type': '2',
            'source': 'channel_pc_web',
            'sec_user_id': sec_user_id,
            'version_code': '160100',
            'version_name': '16.1.0',
            'cookie_enabled': 'true',
            'screen_width': '1920',
            'screen_height': '1080',
            'browser_language': 'zh-CN',
            'browser_platform': 'Win32',
            'browser_name': 'Mozilla',
            'browser_version':ua.replace('Mozilla/',''),
            'browser_online': 'true',
        }
        url = url + parse.urlencode(param)
        _signature = self.get_signature(url)
        url+='&_signature='+quote(_signature)

        headers = {
            "authority": "www.douyin.com",
            "method": "GET",
            "path": str(url).replace('https://www.douyin.com',''),
            "scheme": "https",
            "accept": "application/json, text/plain, */*",
            "accept-language": "zh-CN,zh;q=0.9",
            "cookie": ck,
            "referer": "https://www.douyin.com/user/{sec_user_id}?enter_method=search_result&enter_from=search_result".format(sec_user_id=sec_user_id),
            "user-agent":ua,
            "withcredentials": "true",
        }
        if ck:headers['cookie']=ck
        json_data=self.response(url=url,headers=headers,proxy=proxy)
        return json_data
if __name__ == '__main__':
    webdy=WebDouYin()
    # res = webdy.get_user_videos('MS4wLjABAAAAqLPgx-hHf27EqGEtRQ6YyuQQTmikB5CBO1jXy61yhWKujGd8KO5G8V2vdcLQJAym')
    #print(res)

    # res =webdy.get_user_info('MS4wLjABAAAA2sPiyVAbQc3FsbJJeuyvZkuLjmPRpfKgCvuf41TdjII')
    # print(res)
    # proxy = Proxy.get()
    # print(proxy)
    # proxies = {
    #     "http": "http://" + proxy,
    #     "https": "http://" + proxy
    # }

    ck = webdy.get_ck()
    print(ck)