shop_live_scraper/libs/room_info.py

import requests
import time
import random
import urllib

from libs.proxy import Proxy
from log.print_log import PrintLog
from libs.Xg04 import X_Gorgon


class RoomInfo:
    @staticmethod
    def get_data(room_id):

        domain = random.choice([
            'webcast.amemv.com',
            'webcast-hl.amemv.com',
            'webcast3.amemv.com',
            'webcast3-hl.amemv.com',
            'webcast3-normal-c-lf.amemv.com'
        ])

        app_id = random.choice([
            '1128',
            '2329'
        ])

        # https://webcast-hl.amemv.com/webcast/room/reflow/info/?room_id=6844063184001698568&type_id=0&user_id=81374519744&live_id=1&app_id=1128
        # url = 'http://webcast-hl.amemv.com/webcast/room/reflow/info/?room_id=' + room_id + '&type_id=0&user_id=1&live_id=' + live_id + '&app_id=1128'
        # url = 'http://'+ domain +'.amemv.com/webcast/room/reflow/info/?room_id=' + room_id + '&type_id=0&user_id=1&live_id=1&app_id='+ app_id +'&aid=' + app_id
        url = 'http://' + domain + '/webcast/room/reflow/info/?'

        rticket = str(int(time.time() * 1000))

        query = {
            "room_id": room_id,
            "type_id": "2",  # 0,1
            "user_id": "104255897823",
            "live_id": "1",
            "app_id": app_id
        }

        query_params = urllib.parse.urlencode(query)
        url = url + query_params

        body = ''

        xGorgon = X_Gorgon(query_params, body)

        userAgent = 'okhttp/3.' + str(random.randint(0, 10)) + '.' + str(random.randint(0, 10)) + '.' + str(
            random.randint(1, 10))

        headers = {
            'Host': domain,
            'Connection': 'keep-alive',
            'Cache-Control': 'max-age=0',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': userAgent,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
            "X-SS-REQ-TICKET": rticket,
            "X-Gorgon": xGorgon.get('X-Gorgon'),
            "X-Khronos": xGorgon.get('X-Khronos'),
        }

        retry = 0
        response_json = None

        while True:
            if retry > 10:
                break

            retry += 1

            proxy = Proxy.get()

            proxies = {
                "http": "http://" + proxy,
                "https": "http://" + proxy
            }

            try:
                response = requests.get(
                    url,
                    headers=headers,
                    proxies=proxies,
                    timeout=8
                )
                if (response.status_code == 200) and (response.text is not None) and (response.text != ''):
                    response_json = response.json()

                    if (response_json.get('data') is not None) and (response_json.get('data').get('room') is not None):
                        break
                    else:
                        PrintLog.print(
                            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '数据获取失败！' + '\n'
                            + room_id + '\n'
                            + response.text
                            + Proxy.proxy_info
                        )

                else:
                    PrintLog.print(
                        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '爬取http连接失败！' + str(
                            response.status_code) + '\n'
                        + Proxy.proxy_info + '\n'
                        + room_id + '\n'
                    )
                    time.sleep(1)
            except requests.exceptions.ProxyError as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '代理过期！' + str(e) + '\n'
                    + room_id + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except requests.exceptions.ConnectTimeout as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' ConnectTimeout！' + str(e) + '\n'
                    + room_id + '\n'
                    + Proxy.proxy_info
                )
                Proxy.del_proxy(proxy)
                pass
            except Exception as e:
                PrintLog.print(
                    time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + '请求抛出异常！' + str(e) + '\n'
                    + room_id + '\n'
                    + Proxy.proxy_info
                )
                pass

        return response_json