用python爬取分析淘宝商品信息详解技术篇

目录

• 背景介绍
  • 一、模拟登陆
  • 二、爬取商品信息
    • 1. 定义相关参数
    • 2. 分析并定义正则
    • 3. 数据爬取
  • 三、简单数据分析
    • 1.导入库
    • 2.中文显示
    • 3.读取数据
    • 4.分析价格分布
    • 5.分析销售地分布
    • 6.词云分析
• 写在最后

# 打开图片
def Openimg(img_location):
    img=Image.open(img_location)
    img.show()

# 登陆获取cookies
def Login():  
    driver = webdriver.PhantomJS() 
    driver.get("https://login.taobao.com/member/login.jhtml")
    try:
        driver.find_element_by_xpath("//*[@id="login"]/div[1]/i").click()
    except:
        pass
    time.sleep(3)
    # 执行JS获得canvas的二维码
    JS = "return document.getElementsByTagName("canvas")[0].toDataURL("image/png");"
    im_info = driver.execute_script(JS) # 执行JS获取图片信息
    im_base64 = im_info.split(",")[1]  #拿到base64编码的图片信息
    im_bytes = base64.b64decode(im_base64)  #转为bytes类型
    time.sleep(2)
    with open("./login.png","wb") as f:
        f.write(im_bytes)
        f.close()
    t = threading.Thread(target=Openimg,args=("./login.png",))
    t.start()
    print("Logining...Please sweep the code!
")
    while(True):
        c = driver.get_cookies()
        if len(c) > 20:   #登陆成功获取到cookies
            cookies = {}
            for i in range(len(c)):
                cookies[c[i]["name"]] = c[i]["value"]
            driver.close()
            print("Login in successfully!
")
            return cookies
        time.sleep(1)

# 定义参数
headers = {"Host":"s.taobao.com",
           "User-Agent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0",
           "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
           "Accept-Language":"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
           "Accept-Encoding":"gzip, deflate, br",
           "Connection":"keep-alive"}
list_url = "http://s.taobao.com/search?q=%(key)s&ie=utf8&s=%(page)d"

# 正则模式
p_title = ""raw_title":"(.*?)""       #标题
p_location = ""item_loc":"(.*?)""    #销售地
p_sale = ""view_sales":"(.*?)人付款"" #销售量
p_comment = ""comment_count":"(.*?)""#评论数
p_price = ""view_price":"(.*?)""     #销售价格
p_nid = ""nid":"(.*?)""              #商品唯一ID
p_img = ""pic_url":"(.*?)""          #图片URL

# 数据爬取
key = input("请输入关键字:") # 商品的关键词
N = 20 # 爬取的页数 
data = []
cookies = Login()
for i in range(N):
    try:
        page = i*44
        url = list_url%{"key":key,"page":page}
        res = requests.get(url,headers=headers,cookies=cookies)
        html = res.text
        location = re.findall(p_location,html)
        sale = re.findall(p_sale,html)
        comment = re.findall(p_comment,html)
        price = re.findall(p_price,html)
        nid = re.findall(p_nid,html)
        img = re.findall(p_img,html)
        for j in range(len(title)):
            data.append([title[j],location[j],sale[j],comment[j],price[j],nid[j],img[j]])
        print("-------Page%s complete!--------

"%(i+1))
        time.sleep(3)
    except:
        pass
data = pd.DataFrame(data,columns=["title","location","sale","comment","price","nid","img"])
data.to_csv("%s.csv"%key,encoding="utf-8",index=False)

# 导入相关库
import jieba
import operator
import pandas as pd
from wordcloud import WordCloud
from matplotlib import pyplot as plt

# matplotlib中文显示
plt.rcParams["font.family"] = ["sans-serif"]
plt.rcParams["font.sans-serif"] = ["SimHei"]

# 读取数据
key = "显卡"
data = pd.read_csv("%s.csv"%key,encoding="utf-8",engine="python")

# 价格分布
plt.figure(figsize=(16,9))
plt.hist(data["price"],bins=20,alpha=0.6)
plt.title("价格频率分布直方图")
plt.xlabel("价格")
plt.ylabel("频数")
plt.savefig("价格分布.png")

# 销售地分布
group_data = list(data.groupby("location"))
loc_num = {}
for i in range(len(group_data)):
    loc_num[group_data[i][0]] = len(group_data[i][1])
plt.figure(figsize=(19,9))
plt.title("销售地")
plt.scatter(list(loc_num.keys())[:20],list(loc_num.values())[:20],color="r")
plt.plot(list(loc_num.keys())[:20],list(loc_num.values())[:20])
plt.savefig("销售地.png")
sorted_loc_num = sorted(loc_num.items(), key=operator.itemgetter(1),reverse=True)#排序
loc_num_10 = sorted_loc_num[:10]  #取前10
loc_10 = []
num_10 = []
for i in range(10):
    loc_10.append(loc_num_10[i][0])
    num_10.append(loc_num_10[i][1])
plt.figure(figsize=(16,9))
plt.title("销售地TOP10")
plt.bar(loc_10,num_10,facecolor = "lightskyblue",edgecolor = "white")
plt.savefig("销售地TOP10.png")

# 制作词云
content = ""
for i in range(len(data)):
    content += data["title"][i]
wl = jieba.cut(content,cut_all=True)
wl_space_split = " ".join(wl)
wc = WordCloud("simhei.ttf",
               background_color="white", # 背景颜色
               width=1000,
               height=600,).generate(wl_space_split)
wc.to_file("%s.png"%key)