Python爬虫实战案例：爬取周董《最伟大的作品》31074条评论做词云分析

1. 页面分析
2. 采集程序
3. 对评论分词并绘制词云图

# 专辑网址url = 'https://y.qq.com/n/ryqq/albumDetail/0042cH172YJ0mz'

'//*[@id="comment_box"]/div[4]/ul/li'

'//*[@id="comment_box"]/div[4]/ul/li[1]/div/p/span'

from selenium import webdriverfrom selenium.webdriver.common.by import Byimport time
url = 'https://y.qq.com/n/ryqq/albumDetail/0042cH172YJ0mz'driver = webdriver.Chrome()driver.get(url)driver.implicitly_wait(3)  # 智能化等待：页面渲染加载过程需要时间driver.maximize_window()  # 最大化浏览器for x in range(1, 4000, 2):  # range的范围控制浏览器下拉量    time.sleep(1)    j = x / 9    js = 'document.documentElement.scrollTop = document.documentElement.scrollHeight * %f' % j    driver.execute_script(js)comments = driver.find_elements(By.XPATH,'//*[@id="comment_box"]/div[4]/ul/li')with open('最伟大作品评论.txt', mode='a', encoding='utf-8') as f:    n = 0    for i in comments:        try:            comment = i.find_elements(By.XPATH,'./div/p/span')[0].text            n += 1            print(comment,'第{}条'.format(n))            f.write(comment + '\n')        except:            continuedriver.close()

import jiebafrom wordcloud import WordCloudimport matplotlib.pyplot as pltplt.rcParams['font.sans-serif'] = 'SimHei'
#导入评论文本文件content = open(r'最伟大作品评论.txt',encoding='utf-8')list_all = []for ls in content:    list_temp = ls.strip('\n')    list_all.append(list_temp)
#对list进行遍历,并将其做分词切割后生成词列表word_list =[]exclud = ['，','！','？','《',r'\\','。','的','了',' ']for sentence in list_all:    word_list_temp = jieba.cut(sentence)    for i in word_list_temp:        if i in exclud:            continue        else:            word_list.append(i)word_list = [x for x in word_list if len(x)>=2]print(word_list)
# 绘制词云图comment_word = ' '.join(word_list)cloud=WordCloud(font_path='simhei.ttf',background_color='white',width=1200,height=1200).generate(comment_word)cloud.to_file('最伟大作品评论词云图.png')#保存图片