发布于2019-08-22 17:44 阅读(629) 评论(0) 点赞(21) 收藏(2)
引用
from bs4 import BeautifulSoup
html=requests.get('https://www.cnblogs.com/cate/python/')
soup=BeautifulSoup(html.text,'lxml')
地址获取方法
items=soup.select('div[class="post_item_body"]')
for item in items:
import requests
from bs4 import BeautifulSoup
headers={
'User-Agent':'https://www.cnblogs.com/cate/python/',
'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'cache-control':'max-age=0'
}
html=requests.get('https://www.cnblogs.com/cate/python/')
soup=BeautifulSoup(html.text,'lxml')
items=soup.select('div[class="post_item_body"]')
for item in items:
title=item.select('h3 a[class="titlelnk"]')[0].get_text()
href = item.select('h3 a[class="titlelnk"]')[0]['href']
author=item.select('div a[class="lightblue"]')[0].get_text()
author_home= item.select('div a[class="lightblue"]')[0]['href']
infos=item.select('p[class="post_item_summary"]')[0].get_text().strip('\n').strip(' ')
datas=item.select('div[class="post_item_foot"]')[0].get_text()
datas=datas.split(' ')
# ['\n随风奔跑的少年', '\r\n', '', '', '', '发布于', '2019-07-31', '20:40', '\r\n', '', '', '', '\r\n', '', '', '', '', '', '', '', '评论(0)阅读(4)'] <class 'list'>
#发布时间
time=datas[6]+' '+datas[7]
pinglun=datas[-1].lstrip('评论(').split(')')[0]
read_num = datas[-1].lstrip('评论(').rstrip(')').split('(')[-1]
直接查找属性的类
for mulu in soup.find_all(class_='mulu'):
requests官方文档:http://2.python-requests.org/zh_CN/latest/user/quickstart.html
import requests,re
from bs4 import BeautifulSoup
# div class="hide-featured-badge hide-favorite-badge"
headers={
'user-agent':'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',
}
photos=[]
url="https://www.pexels.com/zh-tw/"
response=requests.get(url,headers=headers)
# print(response.text)
soup=BeautifulSoup(response.text,'lxml')
imgs=soup.select(' div.hide-featured-badge.hide-favorite-badge > article > a > img')#必须保留空格,存在两个属性必须加两个点
# print(imgs)
# print(len(imgs))
for img in imgs:
photo=img.get('src')#获取img的属性src
# print(photo)
if photo.endswith('500'):
photos.append(photo)
# print(photos)
path=r'C:\Users\hh\Desktop\爬虫代码\正式\img'
for item in photos:
data=requests.get(item,headers=headers)
# photo_name=item.split('/')[4]+'.jpg'
photo_name=re.findall('\d+\/(.*?)\?',item)##\转义字符,后面字符原来的意思
# print(photo_name[0])
if photo_name:
fp=open(path+'/'+photo_name[0],'wb')
print(photo_name[0]+'--保存完毕')
fp.write(data.content)
fp.close()
'''
网易云音乐歌手信息抓取
url=https://music.163.com/discover/artist/cat?id=4001&initial=0
https://music.163.com/#/discover/artist/cat?id=4001&initial=0
id=4001 id歌手类别
initial=0
init=[-1,65-90,0]
'''
import requests
from bs4 import BeautifulSoup
import csv
def get_aritists(url):
headers={
'User-Agent':'Mozilla/5.0',
'referer':'https://music.163.com/',
}
r=requests.get(url,headers=headers)
r.encoding=r.apparent_encoding
soup=BeautifulSoup(r.text,'lxml')
for item in soup.find_all('a',attrs={'class':'nm nm-icn f-thide s-fc0'}):
artist_name=item.string.strip()
artist_id=item['href'].replace('/artist?id=','').strip()
print(artist_id,artist_name)
try:
writer.writerow((artist_id,artist_name))
except Exception as e:
print('写入失败')
print(e)
if __name__ == '__main__':
id_list = [1001, 1002, 1003, 2001, 2002, 2003, 6001, 6002, 6003, 7001, 7002, 7003, 4001, 4002, 4003]
init_list = [-1, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
89, 90]
# a=[x for x in range(65,91)]
# print(a)
csvfile=open('music_163_artist.csv','a',encoding='utf-8')
writer=csv.writer(csvfile)
writer.writerow(('artist_id','artist_name'))
for i in id_list:
for j in init_list:
url="https://music.163.com/discover/artist/cat?id={}&initial={}".format(i,j)
# print(url)
get_aritists(url)
# '''
# 票房数据抓取
# 安装pandas,numpy
# - 直接安装anconda(集成环境)
# 利用pandas保存数据
#
# - 抓取中国票房网站的数据
# - url = 'www.cbooo.cn'
# http://www.cbooo.cn/year?year=2019
# '''
import pandas as pd
import requests
from bs4 import BeautifulSoup
url='http://www.cbooo.cn/year?year=2019'
datas=requests.get(url).text
# print(datas)
# 利用bs4对数据进行解析
soup=BeautifulSoup(datas,'lxml')
#获取数据集合
# movies_table=soup.find_all('table',{'id':'tbContent'})
# print(movies_table)
name=[]
# movies=movies_table.findAll('tr')#获取所有的tr
for table in soup.find_all('table',{'id':'tbContent'}):
name.extend([names for names in table.findAll('tr')])
# print(name)
# 获取电影名称
names=[tr.find_all('td')[0].a.get('title') for tr in name[1:]]
# print(names)
#获取电影url地址
hrefs=[tr.find_all('td')[0].a.get('href') for tr in name[1:]]
# print(hrefs)
#获取电影类型
types=[tr.find_all('td')[1].text for tr in name[1:]]
# print(types)
#获取总票房数据
boxoffice=[int(tr.find_all('td')[2].text) for tr in name[1:]]#字符串变为整数类型
# print(boxoffice)
#获取平均票价
mean_price=[int(tr.find_all('td')[3].text) for tr in name[1:]]
# print(mean_price)
# 场均人次
mean_people=[int(tr.find_all('td')[4].text) for tr in name[1:]]
# print(mean_people)
#获取国家和地区
contries=[tr.find_all('td')[5].text for tr in name[1:]]
# print(contries)
# 获取上映时间
times=[tr.find_all('td')[6].text for tr in name[1:]]
# print(times)
def getInfo(url):
datas=requests.get(url)
soup=BeautifulSoup(datas.text,'lxml')
#获取导演
DaoYan=soup.select('dl.dltext dd')[0].get_text()
return DaoYan
directors=[ getInfo(url).strip('\n') for url in hrefs]#去掉\n
# print(directors)
#显示所有列
pd.set_option('display.max_columns', None)
#显示所有行
pd.set_option('display.max_rows', None)
#设置value的显示长度为100,默认为50
# pd.set_option('max_colwidth',100)
df=pd.DataFrame({
'name':names,
'href':hrefs,
'票房':boxoffice,
'总票价':mean_price,
'场均人次':mean_people,
'国家地区':contries,
'time':times,
'导演':directors
})
print(df)
#数据存储
df.to_csv('movies.csv',encoding='utf-8')
作者:fggfg
链接:https://www.pythonheidong.com/blog/article/53258/5f84523c28d9b2c82c56/
来源:python黑洞网
任何形式的转载都请注明出处,如有侵权 一经发现 必将追究其法律责任
昵称:
评论内容:(最多支持255个字符)
---无人问津也好,技不如人也罢,你都要试着安静下来,去做自己该做的事,而不是让内心的烦躁、焦虑,坏掉你本来就不多的热情和定力
Copyright © 2018-2021 python黑洞网 All Rights Reserved 版权所有,并保留所有权利。 京ICP备18063182号-1
投诉与举报,广告合作请联系vgs_info@163.com或QQ3083709327
免责声明:网站文章均由用户上传,仅供读者学习交流使用,禁止用做商业用途。若文章涉及色情,反动,侵权等违法信息,请向我们举报,一经核实我们会立即删除!