shop_live_scraper/libs/proxy.py

import json
import time
import sys

from random import choice
# from libs.db_redis import DbRedis
from log.print_log import PrintLog
sys.path.append("..")
from rds_model.db_redis import DbRedis

class Proxy:

    redis = DbRedis.connect()

    proxy_info = ''

    @staticmethod
    def get():
        while True:
            proxy = Proxy.random_proxy()

            if proxy is not None:
                return proxy

            PrintLog.print('未拿到有效的代理')
            time.sleep(0.1)

    @staticmethod
    def random_proxy():
        key = 'IpProxyHash'
        proxy_dict = Proxy.redis.hgetall(key)

        if (proxy_dict is None) or (len(proxy_dict) == 0):
            return

        proxy_list = list(proxy_dict)

        now = int(time.time())
        while True:
            proxy = choice(proxy_list)#Proxy.choice_proxy()

            if proxy is None:
                return

            proxy_info = proxy_dict.get(proxy)

            if proxy_info is None:
                continue

            Proxy.proxy_info = proxy_info

            proxy_info = json.loads(proxy_info)

            expire_at = int(proxy_info.get('expired_at'))

            # 删除过期的代理
            if expire_at <= now:
                Proxy.redis.hdel(key, proxy)
                proxy_list.remove(proxy)
                continue

            return proxy

    @staticmethod
    def del_proxy(proxy):
        proxy_info = Proxy.redis.hget('IpProxyHash', proxy)
        if proxy_info is None:
            return

        proxy_info = json.loads(proxy_info)
        min_expired_at = proxy_info.get('min_expired_at')

        if min_expired_at is None:
            min_expired_at = 0

        min_expired_at = int(min_expired_at)
        now = int(time.time())

        if min_expired_at >= now:
            return

        # 删除失效的代理
        print('删除失效代理：' + proxy)

    @staticmethod
    def dailiyun_get():
        while True:
            proxy = Proxy.dailiyun_random_proxy()

            if proxy is not None:
                return proxy

            PrintLog.print('未拿到有效的代理')
            time.sleep(0.1)

    @staticmethod
    def dailiyun_random_proxy():
        key = 'IpProxyHashDaiLiYun'
        proxy_dict = Proxy.redis.hgetall(key)

        if (proxy_dict is None) or (len(proxy_dict) == 0):
            return

        proxy_list = list(proxy_dict)

        now = int(time.time())
        while True:
            proxy = choice(proxy_list)#Proxy.choice_proxy()

            if proxy is None:
                return

            proxy_info = proxy_dict.get(proxy)

            if proxy_info is None:
                continue

            Proxy.proxy_info = proxy_info

            proxy_info = json.loads(proxy_info)

            expire_at = int(proxy_info.get('expired_at'))

            # 删除过期的代理
            if expire_at <= now:
                Proxy.redis.hdel(key, proxy)
                proxy_list.remove(proxy)
                continue

            return proxy

    @staticmethod
    def dailiyun_del_proxy(proxy):
        proxy_info = Proxy.redis.hget('IpProxyHashDaiLiYun', proxy)
        if proxy_info is None:
            return

        proxy_info = json.loads(proxy_info)
        min_expired_at = proxy_info.get('min_expired_at')

        if min_expired_at is None:
            min_expired_at = 0

        min_expired_at = int(min_expired_at)
        now = int(time.time())

        if min_expired_at >= now:
            return

        # 删除失效的代理
        print('删除失效代理：' + proxy)

    @staticmethod
    def rola_get():
        while True:
            proxy = Proxy.rola_random_proxy()

            if proxy is not None:
                return proxy

            PrintLog.print('未拿到有效的代理')
            time.sleep(0.1)

    @staticmethod
    def rola_random_proxy():
        key = 'IpProxyHashRola'
        proxy_dict = Proxy.redis.hgetall(key)

        if (proxy_dict is None) or (len(proxy_dict) == 0):
            return

        proxy_list = list(proxy_dict)

        now = int(time.time())
        while True:
            proxy = choice(proxy_list)#Proxy.choice_proxy()

            if proxy is None:
                return

            proxy_info = proxy_dict.get(proxy)

            if proxy_info is None:
                continue

            Proxy.proxy_info = proxy_info

            proxy_info = json.loads(proxy_info)

            expire_at = int(proxy_info.get('expired_at'))

            # 删除过期的代理
            if expire_at <= now:
                Proxy.redis.hdel(key, proxy)
                proxy_list.remove(proxy)
                continue

            return proxy

    @staticmethod
    def rola_del_proxy(proxy):
        proxy_info = Proxy.redis.hget('IpProxyHashRola', proxy)
        if proxy_info is None:
            return

        proxy_info = json.loads(proxy_info)
        min_expired_at = proxy_info.get('min_expired_at')

        if min_expired_at is None:
            min_expired_at = 0

        min_expired_at = int(min_expired_at)
        now = int(time.time())

        if min_expired_at >= now:
            return

        # 删除失效的代理
        print('删除失效代理：' + proxy)