import requests import random import uuid import json import time import urllib import sys from libs.aesgzip import tt_encrypt from rds_model.db_redis import DbRedis from log.print_log import PrintLog from libs.proxy import Proxy from xlog03 import * def get_mc(): def a(): seed = "1234567890ABCDEF" sa = [] for i in range(2): sa.append(random.choice(seed)) salt = ''.join(sa) return salt k = '' for i in range(6): k += a() + ':' return k[:-1] def get_random(i, random_type=1): if random_type == 1: return str(random.randint(1 * 10 ** (i - 1), 1 * 10 ** i - 1)) elif random_type == 8: seed = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" sa = [] for i in range(i): sa.append(random.choice(seed)) salt = ''.join(sa) return salt else: seed = "1234567890abcde" sa = [] for i in range(i): sa.append(random.choice(seed)) salt = ''.join(sa) return salt def get_random_brand_type(): brand_type = get_random(3, random_type=8) + '-' + get_random(2, random_type=8) + '00' return brand_type V1 = '8' V2 = '4' V3 = '8' V4 = '4' channel = 'update' device_type = 'RKK-YZ00' device_brand = 'HUAWEI' class DouYinApi: USER_AGENT = f'com.ss.android.ugc.aweme/{V1}{V2}0 (Linux; U; Android 5.1.1; zh_CN; {device_type}; Build/LMY47V; Cronet/58.0.2991.0)' COMMON_DEVICE_PARAMS = { 'retry_type': 'no_retry', 'ac': '4g', 'channel': channel, 'aid': '1128', 'app_name': 'aweme', 'version_code': f'{V1}{V2}0', 'version_name': f'{V1}.{V2}.0', 'device_platform': 'android', 'ssmix': 'a', 'device_type': device_type, 'device_brand': device_brand, 'language': 'zh', 'os_api': '26', 'os_version': '8.0.0', 'manifest_version_code': f'{V1}{V2}0', 'resolution': '720*1280', 'dpi': '320', 'update_version_code': f'{V1}{V2}02', 'mcc_mnc': '46000' } PROXY = {} def __init__(self, sessionid, proxies): """ :param cid: client id """ self.proxies = proxies self.__cid = '' self.__device_id = '' self.__iid = '' self.__uuid = '' self.__openudid = '' self.__user_agent = '' self.__device_params = {} self.__cookie = { } def common_params_update(self): common_params = { 'click_reason': '0', 'retry_type': 'no_retry', 'ac': '4g', 'channel': channel, 'aid': '1128', 'app_name': 'aweme', 'version_code': f'{V3}{V4}0', 'version_name': f'{V3}.{V4}.0', 'device_platform': 'android', 'ssmix': 'a', 'device_type': device_type, 'device_brand': device_brand, 'language': 'zh', 'os_api': '26', 'os_version': '8.0.0', 'manifest_version_code': f'{V3}{V4}0', 'resolution': '720*1280', 'dpi': '320', 'update_version_code': f'{V3}{V4}02', 'mcc_mnc': '46000' } self.__device_params.update(common_params) def init_device_ids(self, device_id, iid, udid, openudid, cc=None): self.__device_id = device_id self.__iid = iid self.__uuid = udid self.__openudid = openudid self.__user_agent = f'com.ss.android.ugc.aweme/{V1}{V2}0 (Linux; U; Android 5.1.1; zh_CN; {device_type}; Build/LMY47V; Cronet/58.0.2991.0)' device_ids = { 'uuid': udid, 'openudid': openudid, 'version_code': f'{V1}{V2}0', 'version_name': f'{V1}.{V2}.0', 'device_type': device_type, 'device_brand': device_brand, 'manifest_version_code': f'{V1}{V2}0', 'update_version_code': f'{V1}{V2}02' } if device_id and iid: device_ids.update({ 'device_id': device_id, 'iid': iid, }) self.__device_params = self.COMMON_DEVICE_PARAMS.copy() self.__device_params.update(device_ids) if cc: self.__cookie.update(cc) def __get_encrypted_device_info(self, device_id, openudid, udid, clientudid, serial_number, mac, iid): register_info = { "magic_tag": "ss_app_log", "header": { "display_name": "抖音短视频", "update_version_code": int(self.COMMON_DEVICE_PARAMS['update_version_code']), "manifest_version_code": int(self.COMMON_DEVICE_PARAMS['manifest_version_code']), "aid": 1128, "channel": self.COMMON_DEVICE_PARAMS['channel'], "package": "com.ss.android.ugc.aweme", "app_version": self.COMMON_DEVICE_PARAMS['version_name'], "version_code": int(self.COMMON_DEVICE_PARAMS['version_code']), "sdk_version": "2.7.5.8", "os": "Android", "os_version": self.COMMON_DEVICE_PARAMS['os_version'], "os_api": self.COMMON_DEVICE_PARAMS['os_api'], "device_model": self.COMMON_DEVICE_PARAMS['device_type'], "device_brand": self.COMMON_DEVICE_PARAMS['device_brand'], "device_manufacturer": self.COMMON_DEVICE_PARAMS['device_brand'], "cpu_abi": "armeabi-v7a", "build_serial": serial_number, "release_build": "2132ca7_20190320", "density_dpi": self.COMMON_DEVICE_PARAMS['dpi'], "display_density": "xhdpi", "resolution": "1280x720", "language": "zh", "mc": mac, "timezone": 8, "access": "4G", "not_request_sender": 0, "rom": "MIUI-9.11.7", "rom_version": "miui_V11_9.11.7", "openudid": str(openudid), "udid": str(udid), "clientudid": str(clientudid), "serial_number": str(serial_number), "sim_serial_number": [ ], "region": "CN", "tz_name": "Asia/Shanghai", "tz_offset": 28800 }, "_gen_time": str(round(time.time() * 1000)) } if device_id: register_info['header']['device_id'] = str(device_id) if iid: register_info['header']['iid'] = str(iid) register_info['header']['push_sdk'] = '[1, 2, 6, 7, 8, 9]' return tt_encrypt((json.dumps(register_info))) def register_device(self): try: self.common_params_update() udid = '8604' + get_random(11) serial_number = str(uuid.uuid4())[-12:] openudid = '3b22' + str(uuid.uuid4())[-12:] clientudid = str(uuid.uuid4()) mc = get_mc() params = { 'uuid': udid, 'openudid': openudid, '_rticket': str(int(round(time.time() * 1000))) } params.update(self.COMMON_DEVICE_PARAMS) device_register_url = 'https://log.snssdk.com/service/2/device_register/?' + urllib.parse.urlencode(params) headers = { 'User-Agent': DouYinApi.USER_AGENT } d = self.__get_encrypted_device_info(None, openudid, udid, clientudid, serial_number, mc, iid=None) if self.proxies: resp = requests.post(device_register_url, data=d, proxies=self.proxies, headers=headers, verify=False,timeout=10) else: resp = requests.post(device_register_url, data=d, headers=headers, verify=False,timeout=10) cookie = resp.cookies.get_dict() if len(cookie) != 0: self.__cookie.update(cookie) if resp is None: return None resp = resp.json() ids = { 'new_user': resp['new_user'], 'device_id': str(resp['device_id']), 'iid': str(resp['install_id']), 'uuid': udid, 'openudid': openudid, 'serial_number': serial_number, 'clientudid': clientudid, 'mc': mc, 'cookie': urllib.parse.urlencode(self.__cookie) } return ids except Exception as e: print(e) return None def __add_other_params(self, douyin_url, params=None): if params is None: params = {} if not douyin_url.__contains__('?'): douyin_url = douyin_url + '?' common_params = urllib.parse.urlencode(self.__device_params) if douyin_url.endswith('?') or douyin_url.endswith('&'): douyin_url = douyin_url + common_params else: douyin_url = douyin_url + '&' + common_params if len(params) > 0: douyin_url = douyin_url + '&' + urllib.parse.urlencode(params) douyin_url = douyin_url + "&_rticket=" + str(int(round(time.time() * 1000))) + "&ts=" + str(int(time.time())) return douyin_url def get_user_post(self, user_id, max_cursor, count): """获取用户作品 :param user_id: 用户ID :param max_cursor: 用于分页，第1页是0，后1页是上1页请求的时候返回的max_cursor :param count: 返回视频的条数 :return: """ self.common_params_update() params = { 'user_id': str(user_id), 'max_cursor': str(max_cursor), 'count': str(count) } douyin_url = 'https://aweme.snssdk.com/aweme/v1/aweme/post/' return self.__http_get(douyin_url, params) def __http_get(self, url, query_params=None): if query_params is None: query_params = {} url = self.__add_other_params(url, query_params) sign = self.__get_sign(url, query_params) headers = self.__get_headers(sign) if self.proxies: resp = requests.get(url, headers=headers, cookies=self.__cookie, proxies=self.proxies, verify=False, timeout=8) else: resp = requests.get(url, headers=headers, cookies=self.__cookie, verify=False) cookie = resp.cookies.get_dict() if len(cookie) != 0: self.__cookie.update(cookie) return resp.json() def __get_sign(self, url, form_params=None): stub = '' if form_params: a = urllib.parse.urlencode(form_params) stub = hashlib.md5(a.encode('utf-8')).hexdigest() ts = int(time.time()) ppp = url[url.index('?') + 1:] s = getXGon(ppp, stub, urllib.parse.urlencode(self.__cookie)) gorgon = xGorgon(ts, strToByte(s)) sign = { 'X-Khronos': str(ts), 'X-Gorgon': gorgon, 'X-Pods': '' } #print(gorgon) if stub: sign.update({ 'X-SS-STUB': stub.upper() }) return sign def __get_headers(self, sign=None): if sign is None: sign = {} headers = { 'User-Agent': self.__user_agent, 'X-SS-REQ-TICKET': str(round(time.time() * 1000)), } headers.update(sign) return headers if __name__ == '__main__': import warnings start_time = time.time() warnings.filterwarnings("ignore") while True: server_time = int(time.time()) current_time = time.time() if current_time - start_time > 300: print( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ' 主方法执行终止' ) sys.exit(0) # key = 'DOUYIN_SCRAPE_DID_IID_TTREQ_1221' # rdid = DbRedis.douyin_connect().rpop(key) # if rdid: # result = rdid.split('@@@') # else: # time.sleep(2) # continue proxy = Proxy.get() proxies = { "http": "http://" + proxy, "https": "http://" + proxy } douApi = DouYinApi('', proxies) result = douApi.register_device() if result is None: time.sleep(1) continue errorn = 0 suc_f = 0 for index in range(1000): try: ukey = 'BrandLiveData.DouyinUserVideo' users = DbRedis.douyin_connect().rpop(ukey) if users is None: time.sleep(10) continue user = json.loads(users) user_id = user.get('uid') re_times = int(user.get('re_times')) + 1 user.update({ 're_times':re_times }) users = json.dumps(user) # device_id, iid, udid, openudid, cookie, V1, V2, device_type, device_brand = result[0], result[1], result[3], result[2], result[4], result[8], result[9], result[10], result[11] device_id, iid, udid, openudid, cookie = result['device_id'], result['iid'], result['uuid'], \ result['openudid'], result['cookie'] if device_id == '0': print('device失败') break douApi.init_device_ids(device_id, iid, udid, openudid) videos = douApi.get_user_post(user_id, 0, 20) awemes = videos.get('aweme_list') stime = time.strftime("%H:%M:%S", time.localtime()) if awemes: PrintLog.print(stime+" 成功"+str(index)+" "+str(user_id)+" V: "+str(V1+V2+'0')+" "+str(device_type)) print(stime+" 成功"+str(index)) dkey = 'BrandLiveData.DouyinUserVideoResponsePython' data = str(user_id) + '@@@' + json.dumps(videos) DbRedis.douyin_connect().lpush(dkey, data) else: PrintLog.print(stime+" 失败"+str(index)+' '+ str(user_id)) print(stime+" 失败"+str(index)) errorn = errorn + 1 if re_times<1: ukey = 'BrandLiveData.DouyinUserVideo' DbRedis.douyin_connect().lpush(ukey, users) if errorn>2: break except Exception as e: print( time.strftime("%H:%M:%S", time.localtime()) + ' 请求抛出异常！行号：' + str(e.__traceback__.tb_lineno) + ' 错误:' + str(e) + "\n" ) break