利用python爬取51job工作岗位信息，并导入excel
本帖最后由 ontheway168 于 2020-5-16 00:13 编辑requests爬虫.py文件
import re
import requests
from excel_write import excel_init
workbook, worksheet = excel_init()
#获取总页码数
def get_pages():
#模拟一个虚拟网站
user_header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
#对应地区测试工程师搜索网址
web_url = 'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
res = requests.get(web_url, headers=user_header)
res.encoding = 'gbk'
pages = int(re.findall('<span class="td">共(.*?)页，到第</span>', res.text, re.S).strip())
return pages
row = 1
for page in range(1, get_pages()+1):
#模拟一个虚拟网站
user_header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
web_url = f'https://search.51job.com/list/030800,000000,0000,00,9,99,%25E6%25B5%258B%25E8%25AF%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,{page}.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
res = requests.get(web_url, headers=user_header)
res.encoding = 'gbk'
# print(res.text)
'''
<div class="el">
<p class="t1 tg1">
<em class="check" name="delivery_em"></em>
<input class="checkbox" type="checkbox" name="delivery_jobid" value="120608298" jt="0" style="display:none">
<span>
<a target="_blank" title="测试工程师" href="https://jobs.51job.com/dongguan/120608298.html?s=01&t=0">
测试工程师 </a>
</span>
<img width="30" height="20" src="//img01.51jobcdn.com/im/jobs/tag_xiaozhao.png?20190903" alt="校招">
</p>
<span class="t2"><a target="_blank" title="东莞新能德科技有限公司" href="https://jobs.51job.com/all/co5796557.html">东莞新能德科技有限公司</a></span>
<span class="t3">东莞</span>
<span class="t4"></span>
<span class="t5">05-15</span>
</div>
'''
lines = re.findall('<div class="el">(.*?)</div>', res.text, re.S)
for line in lines:
temp = re.findall('<a target="_blank" title="(.*?)" href', line, re.S)
job = temp.strip()
worksheet.write(row, 0, job)
company = temp.strip()
worksheet.write(row, 1, company)
address = re.findall('<span class="t3">(.*?)</span>', line, re.S).strip()
worksheet.write(row, 2, address)
salary = re.findall('<span class="t4">(.*?)</span>', line, re.S).strip()
worksheet.write(row, 3, salary)
datetime = re.findall('<span class="t5">(.*?)</span>', line, re.S).strip()
worksheet.write(row, 4, datetime)
row += 1
# print(job,company,address,salary,datetime)
workbook.save('./51job岗位信息.xls')
excel_write.py文件
import xlwt
#创建一个excel文件，创建表单，写
def excel_init():
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('51job岗位信息')
colNames = ['岗位名称', '公司名称', '地址', '薪资', '发布时间']
for i in range(len(colNames)):
worksheet.write(0, i, colNames)#(行号，列号，内容)，行列号从0开始
return workbook, worksheet
{:10_257:}{:10_257:}可以嫖走学习吗
页:
[1]