链家租房数据爬虫

利用xpath和正则对链家网站北京市东城区、海淀区、丰台区的租房房源前50页数据进行爬取。数据包括链家网站链家网站北京市东城区、海淀区、丰台区的租房房源数据共4500条，包括房源信息标题、区域、地点、房屋名称、建筑面积、朝向、户型、价格、详情页链接等信息。

1.  
   import requests
2.  
   import json
3.  
   from lxml import etree
4.  
   import re
5.  
    
6.  
   def get_info(zone):
7.  
   for i in range(1, 51):
8.  
   headers = {
9.  
   "user-agent": "# 自己的user-agent"
10.  
    }
11.  
     
12.  
    url = 'https://bj.lianjia.com/zufang/' zone '/pg' str(i) '/#contentList'
13.  
    url_total = 'https://bj.lianjia.com/zufang/' zone
14.  
    # get or post 看请求Headers
15.  
    html = requests.get(url, headers=headers)
16.  
    html = html.text
17.  
    soup = etree.HTML(html)
18.  
     
19.  
    # 标题
20.  
    title = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[1]/a//text()')
21.  
    title_new = []
22.  
    for i in title:
23.  
    i = i.strip('\n') # 删除换行符
24.  
    i = i.strip(' ') #删除空格
25.  
    title_new.append(i)
26.  
    # print(title_new)
27.  
     
28.  
    # 所在区域（海淀、丰台、东城）
29.  
    zones = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[1]//text()')
30.  
     
31.  
    # 地点
32.  
    place = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[2]//text()')
33.  
    # print(place)
34.  
     
35.  
    # 具体名称
36.  
    name = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/a[3]//text()')
37.  
    # print(name)
38.  
     
39.  
    # 面积
40.  
    size = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]//text()[5]')
41.  
    size_new = []
42.  
    for i in size:
43.  
    i = i.strip('\n') # 删除换行符，但是只能删除第一个换行符
44.  
    i = i.replace('\n', '') # 利用replace将换行符替换
45.  
    i = i.strip(' ')
46.  
    size_new.append(i)
47.  
    # print(size_new)
48.  
     
49.  
    # 朝向
50.  
    orientation = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/text()[6]')
51.  
    orientation_new = []
52.  
    for i in orientation:
53.  
    i = i.strip(' ')
54.  
    orientation_new.append(i)
55.  
    # print(orientation_new)
56.  
     
57.  
    # 户型
58.  
    type = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/p[2]/text()[7]')
59.  
    type_new = []
60.  
    for i in type:
61.  
    i = i.strip('\n')
62.  
    i = i.strip(' ')
63.  
    type_new.append(i)
64.  
    # print(type_new)
65.  
     
66.  
    # 价格
67.  
    price = soup.xpath('/html/body/div[3]/div[1]/div[5]/div[1]/div[1]/div/div/span/em//text()')
68.  
    # print(price)
69.  
     
70.  
    详情页链接
71.  
    house_url = re.findall('<a class="twoline" target="_blank" href="https://blog.csdn.net/m0_69208651/article/details/(/zufang/.*?.html)">', html)
72.  
    # print(house_url)
73.  
     
74.  
    # 将爬取信息存入excel
75.  
    for i in range(len(title)):
76.  
    f = open('zufang.csv', 'a', encoding='utf8')
77.  
    f.write('{},{},{},{},{},{},{},{},{}\n'.format(title_new[i], zones[i], place[i], name[i], size_new[i], orientation_new[i], type_new[i], price[i], url_total house_url[i]))
78.  
    f.close()
79.  
     
80.  
    if __name__ == '__main__':
81.  
    f = open('zufang.csv', 'a', encoding='utf8')
82.  
    f.write('title,zone,place,name,size,orientation,type,price,link\n') # 写入表头
83.  
    f.close()
84.  
    for zone in ('dongcheng', 'haidian', 'fengtai'):
85.  
    get_info(zone)