shop_live_scraper/libs/live_promoti...

import requests
import time
import random

from libs.proxy import Proxy
from log.print_log import PrintLog


class LivePromotions:

    @staticmethod
    def get_data(uid, room_id):
        # if random.randint(0, 2) > 0:
        #     uid = '1'

        # url = 'https://lianmengapi.snssdk.com/live/promotions/?author_id=' + uid + '&room_id=' + room_id + '&aid=1128'
        url = 'http://lianmengapi.snssdk.com/live/livePromotions/?author_id=' + uid + '&room_id=' + room_id + '&os_api=22&device_type=vivo%20X20&device_platform=android&ssmix=a&manifest_version_code=730&dpi=320&version_code=730&app_name=aweme&version_name=7.3.0&resolution=900*1600&language=zh&device_brand=vivo%20&app_type=normal&ac=wifi&update_version_code=7302&aid=1128&channel=xiaomi&mcc_mnc=46007'

        headers = {
            'Host': 'lianmengapi.snssdk.com',
            'Connection': 'keep-alive',
            'Cache-Control': 'max-age=0',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3908.2 Mobile Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-Mode': 'navigate',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
        }

        retry = 0
        response = None

        while True:
            if retry > 10:
                break

            retry += 1

            proxy = Proxy.get()

            proxies = {
                "http": "http://" + proxy,
                "https": "http://" + proxy
            }

            try:
                response = requests.get(
                    url,
                    headers=headers,
                    proxies=proxies,
                    timeout=10
                )

                if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
                    break
                else:
                    PrintLog.print(
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '爬取http连接失败！' + str(response.status_code) + '\n'
                        + Proxy.proxy_info + '\n'
                        + uid + ' ' + room_id + '\n'
                    )
                    time.sleep(1)
            except requests.exceptions.ProxyError as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '代理过期！' + str(e) + '\n'
                    + uid + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except requests.exceptions.ConnectTimeout as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' ConnectTimeout！' + str(e) + '\n'
                    + uid + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except Exception as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求抛出异常！' + str(e) + '\n'
                    + uid + ' ' + room_id + '\n'
                    + Proxy.proxy_info
                )
                pass

        return response