shop_live_scraper/douyin_video...

import json
import sys
import threading
import time
import warnings

from libs.proxy import Proxy
from log.print_log import PrintLog
from rds_model.db_redis import DbRedis
from rds_model.rds_user_video_list import RdsUserVideoList
from web_cookie import Cookie
from web_dy import *

# -- coding: utf-8 --**
start_time = int(time.time())

def set_score(flag):
    rds = RdsUserVideoList()

    if flag == 'success':
        data_score = rds.get_score()
        if data_score is None:
            data_score = '1@@@1@@@0'
        else:
            data_score = data_score.split('@@@')
            total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
            success = success + 1
            data_score = str(total) + '@@@' + str(success) + '@@@' + str(fail)
        rds.record_score(data_score)
    elif flag == 'fail':
        data_score = rds.get_score()
        if data_score is None:
            data_score = '1@@@0@@@1'
        else:
            data_score = data_score.split('@@@')
            total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
            fail = fail + 1
            data_score = str(total) + '@@@' + str(success) + '@@@' + str(fail)
        rds.record_score(data_score)
    elif flag == 'all':
        data_score = rds.get_score()
        if data_score is None:
            data_score = '1@@@0@@@0'
        else:
            data_score = data_score.split('@@@')
            total, success, fail = int(data_score[0]), int(data_score[1]), int(data_score[2])
            total = total + 1
            data_score = str(total) + '@@@' + str(success) + '@@@' + str(fail)
        rds.record_score(data_score)
    return None

def get_signature(url=None,method='_signature'):
    # with open('/mnt/kwai_data_center_test_scraper/signature.js', 'r', encoding='utf-8') as f:
    with open('/mnt/shop_live_scraper_shen/signature.js', 'r', encoding='utf-8') as f:
        b = f.read()
    
    c = execjs.compile(b)
    
    # url=url.replace('%28','(').replace('%29',')').replace('%2C',',')
    d = c.call(method, url.replace('\n',''))
    
    return d

def get_ua_ck():
    ua_list=[
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3870.400 QQBrowser/10.8.4405.400"
    ]
    ck_list=[
        'ttwid=1%7CTVzdM0P0u-8dtsmh6c-EaQEtBoTSOs_MG85FAg07AbA%7C1631502013%7C66442d8594de8e93ad18b73f3dfe0c94ed864c3d932824bcde9918b5be172321; passport_csrf_token=866923f1a32045fd82e47053158402a2',
        'ttwid=1%7CGPDDu9-w3RGs2Pcd0wRlvLYoktpDt-v8LP5ZMyb1NBM%7C1630319594%7Cffb8de47e6da87dcfd76349b5ad34aa1f9b9d4332261a3a8436b932a893366c1; passport_csrf_token=79284b8777a7a54f3066cefef9af539e',
        'ttwid=1%7CGsfqc7NpdOg4N-U-VX7Q77KsWjVTZ7gxLNifsisj8YE%7C1631618570%7Cafbb13a27fd1c2d7a8245454b1e0d7cd654d80848a320933a25d9ef77638c18c; passport_csrf_token=84911c9af94040a99cc10416bd27533d',
        'ttwid=1%7C82FGr05YUOReYUB301ao_erqOQ3ilbXZdEy0tkMsdXY%7C1631863641%7C1dcebe643a96f00841a3b490db60de886bfe07ff3d276e509717abc4e1681ba6; passport_csrf_token=494ae3fffe00328101fd40e050ce49db',
        'ttwid=1%7CwfnX3T9LY4_60iGoQNzyqYe5ahILFeRxfMuZ1pdgXf8%7C1632724192%7Cb613fddc0b533d5578dad4d5f9290705fdc6432aa854d492f4761d164dd3fdd5; passport_csrf_token=4a8afba333103b033e537003b72ee91b'
    ]
    return random.choice(ua_list),random.choice(ck_list)

def get_user_videos(sec_user_id, max_cursor=0, count=20):

    ua,ck=get_ua_ck()
    # ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    
    url='https://www.douyin.com/aweme/v1/web/aweme/post/?'
    param={
        'device_platform': 'webapp',
        'aid': '6383',
        'channel': 'channel_pc_web',
        'sec_user_id': sec_user_id,
        'max_cursor': str(max_cursor),
        'count': str(count),
        'publish_video_strategy_type': '2',
        'version_code': '170400',
        'version_name': '17.4.0',
        'cookie_enabled': 'true',
        'screen_width': '1920',
        'screen_height': '1080',
        'browser_language': 'zh-CN',
        'browser_platform': 'Win32',
        'browser_name': 'Mozilla',
        'browser_version':ua.replace('Mozilla/',''),
        'browser_online': 'true',
        "source" : "channel_pc_web"
    }

    url = url + parse.urlencode(param)
    
    _signature = get_signature(url)
    url+='&_signature='+quote(_signature)

    ck = Cookie.get()
    if ck is None:
        print('获取cookie失败')
        return None
    
    headers = {
        "authority": "www.douyin.com",
        "method": "GET",
        "path": str(url).replace('https://www.douyin.com',''),
        "scheme": "https",
        "accept": "application/json, text/plain, */*",
        # "accept-encoding": "gzip, deflate, br",
        "accept-language": "zh-CN,zh;q=0.9",
        "cookie": ck,
        "referer": "https://www.douyin.com/user/{sec_user_id}?enter_method=search_result&enter_from=search_result".format(sec_user_id=sec_user_id),
        "user-agent":ua,
        "withcredentials": "true",
        "sec-ch-ua" : '"Google Chrome";v="93", " Not;A Brand";v="99", "Chromium";v="93"',
        "sec-ch-ua-mobile" : "?0",
        "sec-ch-ua-platform" : "Windows",
        "sec-fetch-dest" : "empty",
        "sec-fetch-mode" : "cors",
        "sec-fetch-site" : "same-origin"
    }

    if ck:
        headers['cookie']=ck

    retry = 0
    response_json = None

    while True:
        if retry > 20:
            Cookie.del_cookie(ck)
            break

        retry += 1

        # proxy = Proxy.get()
        proxy = Proxy.rola_get()

        proxies = {
            "http": "http://" + proxy,
            "https": "http://" + proxy
        }

        try:
            response = requests.get(
                url,
                headers=headers,
                proxies=proxies,
                timeout=8
            )
            if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
                # print(response)
                response_json = response.json()

                if (response_json.get('aweme_list') is not None):
                    print(
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                        + ' 数据获取成功！' 
                        + '\n'
                        + str(sec_user_id) 
                    )
                    break
                else:
                    print(
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                        + ' 数据获取失败！' 
                        + '\n'
                        + str(sec_user_id) 
                        + '\n'
                        + response.text
                        + Proxy.proxy_info
                    )
            else:
                print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                    + ' 爬取http连接失败！' 
                    + str(response.status_code) 
                    + '\n'
                    + Proxy.proxy_info 
                    + '\n'
                    + str(sec_user_id) 
                    + '\n'
                    + '爬取结果：' + str(response)
                    + '\n'
                )
                time.sleep(1)
        except requests.exceptions.ProxyError as e:
            print(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                + ' 代理过期！' 
                + str(e) 
                + '\n'
                + str(sec_user_id) 
                + '\n'
                + Proxy.proxy_info
            )
            # Proxy.del_proxy(proxy)
            Proxy.rola_del_proxy(proxy)
            pass
        except requests.exceptions.ConnectTimeout as e:
            print(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                + ' ConnectTimeout！' 
                + str(e) 
                + '\n'
                + str(sec_user_id) 
                + '\n'
                + Proxy.proxy_info
            )
            # Proxy.del_proxy(proxy)
            Proxy.rola_del_proxy(proxy)
            pass
        except Exception as e:
            print(
                time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                + ' 请求抛出异常！' 
                + str(e) 
                + '\n'
                + str(e.__traceback__.tb_lineno)
                + '\n'
                + str(sec_user_id) 
                + '\n'
                + Proxy.proxy_info
            )
            pass
        
    return response_json

def scrape():
        
    rds = RdsUserVideoList()

    while True:
        user_info = rds.get_request_param()
        if user_info is None:
            time.sleep(60)
            continue
            # return None

        sec_user_id = str(user_info)

        print(
            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
            + ' ' 
            + str(sec_user_id)
        )

        try:
            videos = get_user_videos(sec_user_id=sec_user_id,max_cursor=0,count=1)
        
            if videos is None:
                # rds.push_request_id(sec_user_id)
                print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
                    + ' 数据获取失败！响应数据为空！' 
                    + '\n'
                    + str(sec_user_id) 
                    + '\n'
                )
                # sys.exit(0)

            if isinstance(videos, dict):
                awemes = videos.get('aweme_list')
            else:
                # print(videos)
                awemes = None

            if awemes:
                set_score('success')
                data = str(sec_user_id) + '@@@' + json.dumps(videos)
                rds.push_data_list(data)
            else:
                set_score('fail')

        except Exception as e:
            set_score('fail')
            rds.push_request_id(sec_user_id)
            print(
                time.strftime("%H:%M:%S", time.localtime()) 
                + ' ' 
                + str(sec_user_id)
                + '数据异常：' 
                + str(e)
            )
        
        current_time = time.time()
        if current_time - start_time > 3600:
            sys.exit(0)

    # sys.exit(0)

if __name__ == '__main__':
    print("主方法开始执行")
    # 并行线程数
    threading_count = int(sys.argv[1])
    # num = int(sys.argv[2])

    rds = RdsUserVideoList()
    warnings.filterwarnings("ignore")

    print(
        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 
        + ' ' 
        + ' 开始执行，用户队列长度：' 
        + str(rds.get_len())
    )

    while True:
        sys.stdout.flush()

        current_time = time.time()
        if current_time - start_time > 3600:
            sys.exit(0)
        
        # 减去主线程
        active_count = threading.active_count() - 1
       
        increment = threading_count - active_count
        
        if increment > 0:
            task = threading.Thread(target=scrape, args=())
            task.start()  # 准备就绪, 等待cpu执行
            # increment = increment - 1
        
        time.sleep(1)