本文介绍python的requests库抓取搜狐历史频道，即下面连接：

http://history.sohu.com/?spm=smpc.news-home.history-nav.2.1565598359654B62Bf5B

注意：

• 此连接有反爬虫机制，要加上cookie
• 此网页是动态加载，但不是ajax，返回的不是标准的json数据格式

# coding=utf-8
import requests,time,json

g_url='http://v2.sohu.com/integration-api/mix/region/6282?size=25&adapter=pc&secureScore=50&page=%s&pvId=1565660260783R2WURQZ&requestId=180508174953J48I_1565660260783&callback=jQuery11240005847590633839994_1565660260185&_=1565660260186'
page=1
headers={
    'Cookie':'SUV=180508174953J48I; gidinf=x099980109ee0dcaa6cfc80400002e3662edd8b4046d; beans_mz_userid=fybEf0w6qeeq; _muid_=1553136051199963; IPLOC=CN2224; reqtype=pc; t=1565660300977',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
    }

def getlist(url):
    global page
    print('++'*50)
    #发送get请求
    with requests.get(url,headers=headers) as resp:
        #打印状态码，英文提示
        print(resp.status_code,resp.reason)
        #返回的编码
        print(resp.encoding)
        #请求的url
        print(resp.url)
        #响应头，特殊的字典类型，key不区分大小写
        print(resp.headers)

        jsonstr=resp.text
        jsonstr=jsonstr[jsonstr.index('(')+1:jsonstr.index(')')]
        # print(jsonstr)
        try:
            jsonobj=json.loads(jsonstr,encoding='utf-8')
        except:
            print('json格式转换错误，停止运行')

            time.sleep(3)
            page+=10
            getlist(g_url%page)
            return
        print(type(jsonobj))
        data=jsonobj.get('data')
        for item in data:
            # print(type(item))
            article_url=item.get('url')
            if article_url:
                print('title:',item.get('title'))
                print('url:',article_url)


    time.sleep(3)
    page+=10
    getlist(g_url%page)
if __name__ == "__main__":
    getlist(g_url%page)

原文出处：/show_1EF3sAJYCkqg.html