掏出了以前的小练习；

现在开始，每天复习下以前的爬虫练习，争取发现新的问题和可以优化的地方。

# -*- coding:utf-8 -*-import requestsimport chardetimport csvfrom lxml import etreeimport redef get_page(url):    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'    header = {
   'User-Agent':user_agent}    r = requests.get(url,headers=header)    r.encoding = chardet.detect(r.content)['encoding']    page = r.text    return pagedef parse_data(page):    result = []    html = etree.HTML(page)    next_url = html.xpath('//a[@class="nxt"]/@href')    if len(next_url) > 0:        next_url = next_url[0]        print(next_url)    sites = html.xpath('//*[@class="xld xlda cl"]')        for site in sites:        title = site.xpath('.//a[@class="xi2"]/text()')[0]        author = site.xpath('.//p[@class="xg1"]/a/text()')[0]        theme = site.xpath('.//strong[@class="xi2"]/text()')[0]        r = site.xpath('./dl/dd[2]/p[2]/text()')[0]        sub_num,com_num =r.split(',')        com_num = com_num.strip()        sub_num = sub_num.strip()        content = (title,author,theme,sub_num,com_num)        result.append(content)    return result,next_urldef main():    url = 'http://bbs.fishc.org/forum.php?mod=collection'    results = []    page = get_page(url)    result,next_url = parse_data(page)    results.extend(result)    q = True    while q:        if next_url:            page = get_page(next_url)            result,next_url = parse_data(page)            results.extend(result)        else:            q = False    headers = ['title','author','theme','sub_num','com_num']    with open(r'taotie.csv','w',encoding = 'utf-8') as f:        f_csv = csv.writer(f)        f_csv.writerow(headers)        try:            f_csv.writerows(results)        except UnicodeDecodeError as e:            print(e)                        if __name__ =="__main__":    main()