作为⼀个python⼩⽩，在下⾯的问题中出错：
1.因为⾖瓣页⾯的数据加载涉及到异步加载，所以需要通过浏览器获取到真正的⽹页链接。
2.将字典转化为DataFrame以后写⼊.csv⽂件。DataFrame是⼀个表单⼀样的数据结构。
3.从⽹页获取的json数据的处理。
代码：
import re
import requests
from bs4 import BeautifulSoup
import time
import random
import string
import logging
import json
import jsonpath
import pandas as pd
import pdb
User_Agents = [
'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
]
class DoubanSpider(object):
'''⾖瓣爬⾍'''
def __init__(self):
# 基本的URL
self.base_url = 'https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_' + 'start={start}' self.full_url = self.base_url
self.tv_detailurl = 'https://movie.douban.com/j/subject_abstract?subject_id='
def download_tvs(self, offset):
# offset控制⼀次下载的量，resp返回的响应体
self.headers = {'User-Agent': random.choice(User_Agents)}
self.full_url = self.base_url.format(start=offset)
resp = None
try:
resp = requests.get(self.full_url, headers=self.headers)
except Exception as e:
print(logging.error(e))
return resp
def get_tvs(self, resp):
# resp响应体
# movies爬取到的电影信息
print('get_tvs')
print(resp)
tv_urls = []
if resp:
if resp.status_code == 200:
html = resp.text
unicodestr = json.loads(html)
tv_list = unicodestr['subjects']
for item in tv_list:
data = re.findall(r'[0-9]+', str(item['url']))
tv_urls.append(self.tv_detailurl + str(data[0]))
print('tv_urls')
return tv_urls
return None
def download_detailtvs(self, tv_urls):
tvs = []
for item in tv_urls:
self.headers = {'User-Agent': random.choice(User_Agents)}
resp = requests.get(item, headers=self.headers)
html = resp.text
unicodestr = json.loads(html)
tvs.append(unicodestr['subject'])
return tvs
def main():
spider = DoubanSpider()
offset =0
data = {'title':[], 'types':[], 'directors':[], 'actors':[], 'release_year':[], 'region':[], 'star':[], 'episodes_count':[],'rate':[]}
tv_file = pd.DataFrame(data)
tv_file.to_csv('res_url.csv',mode='w',index=None)
while True:
reps = spider.download_tvs(offset)
tv_urls = spider.get_tvs(reps)
tvss = spider.download_detailtvs(tv_urls)
for tvsss in tvss:
'''#pdb.set_trace()
tvsss=re.sub(r'\\\','',tvsss)
tvsss = re.sub(r'\\'', '\\"', tvsss)
tvsss = re.sub(r'\\'', '\\"', tvsss)
#将short_comment去掉
tvsss = re.sub(r'(\\"short_comment\\").*(\\"directors\\")', '\\"directors\\"',tvsss)
#将true,false改为"True
tvsss = re.sub(r'True', '\\"True\\"', tvsss)
tvsss = re.sub(r'False', '\\"False\\"', tvsss)
#给所有的list加上双引号
print(tvsss)
#将: [转化为: "[
tvsss = re.sub(r': \\[', ': "[', tvsss)
#jiang ],zhuanhuawei ]
tvsss=re.sub(r'\\],',']
# 以上正确
print(tvsss)
#将director的内容改为单引号
r1 = re.findall(r'(?<=directors": "\\[).*?(?=\\]\\")', tvsss)
#正确
if r1:
r2 = re.sub(r'\\"', '\\'', r1[0])
r3 = re.sub(r'\\"', '\\'', r2)
tvsss = re.sub(r'(?<=directors\\": \\"\\[).*?(?=\\]\\")', r3, tvsss)
#zhengque
#将actors的内容改为单引号
print(tvsss)
r1 = re.findall(r'(?<=actors\\": \\"\\[).*?(?=\\]\\")', tvsss)
print("actors")
print(r1)
if r1:
r2 = re.sub(r'\\"', '\\'', r1[0])
r3 = re.sub(r'\\"', '\\'', r2)
tvsss = re.sub(r'(?<=actors\\": \\"\\[)[\\s\\S]*?(?=\\]\\")', r3, tvsss)
#将剧情types改为单引号
print(tvsss)
r1 = re.findall(r'(?<=types": "\\[).*?(?=\\]\\")', tvsss)
if r1:
r2 = re.sub(r'\\"', '\\'', r1[0])
r3 = re.sub(r'\\"', '\\'', r2)
tvsss = re.sub(r'(?<=types\\": \\"\\[).*?(?=\\]\\")', r3, tvsss)
# 正确
#将⼆维的数据转化为⼀维的
types=str(tvs['types'])
actor = str(tvs['actors'])
director = str(tvs['directors'])
types=re.sub(r'\\'','',types)
actor = re.sub(r'\\'', '', actor)
director = re.sub(r'\\'', '', director)
types = re.sub(r'\\'', '', types)
actor= re.sub(r'\\'', '', actor)
director = re.sub(r'\\'', '', director)
types=types.strip('[]')
actor=actor.strip('[]')
director=director.strip('[]')
data2={'title':tvs['title'], 'types':types, 'directors':director, 'actors':actor, 'release_year':tvs['release_year'], 'region':tvs['region'], 'star':tvs['star'], 'episodes_count':tvs['episodes_count'],'rate':tvs['rate']} print(data2)
tv_file=pd.DataFrame(data2,index=[0])
#tv_file = pd.DataFrame(data)
# pdb.set_trace()
tv_file.to_csv('res_url.csv', mode='a', index=None,header=None)
offset += 20
id = offset
# 控制访问速速
time.sleep(10)
if __name__ == '__main__':
main()下载本文
