shop_live_scraper/libs/live_commodi...

import requests
import time
import random

from libs.proxy import Proxy
from log.print_log import PrintLog

class LiveCommodityDetailV1:

    @staticmethod
    def get_data(product_id):
        url = 'http://ec.snssdk.com/product/lubanajaxstaticitem?id=' + product_id + '&page_id=&scope_type=5&item_id=&b_type_new=0'

        userAgentList = [
            'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36 SE 2.X MetaSr 1.0',
            'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3908.2 Mobile Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:92.0) Gecko/20100101 Firefox/92.0',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
            'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
            'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201',
            'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)'
        ]

        headers = {
            'Host': 'ec.snssdk.com',
            'Connection': 'keep-alive',
            'Cache-Control': 'max-age=0',
            'User-Agent': random.choice(userAgentList),
            'Referer': 'https://haohuo.jinritemai.com/',
            'Accept': 'application/json, text/plain, */*',
        }

        while True:

            proxy = Proxy.get()

            proxies = {
                "http": "http://" + proxy,
                "https": "http://" + proxy
            }

            try:
                response = requests.get(
                    url,
                    headers=headers,
                    proxies=proxies,
                    timeout=10
                )
                if response.text is not None:
                    break
                else:
                    print(response)
                    print('爬取http连接失败！')
                    time.sleep(1)
            except requests.exceptions.ProxyError as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '代理过期！' + str(e) + '\n'
                    + product_id + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except requests.exceptions.ConnectTimeout as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' ConnectTimeout！' + str(e) + '\n'
                    + product_id + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except Exception as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 请求抛出异常！' + str(e) + '\n'
                    + product_id + '\n'
                    + Proxy.proxy_info
                )
                pass

        return response.text