Python使用Selenium模块模拟浏览器抓取斗鱼直播间信息示例_Python

本文实例讲述了Python使用Selenium模块模拟浏览器抓取斗鱼直播间信息。分享给大家供大家参考，具体如下：
				?

									import time

									from multiprocessing import Pool

									from selenium import webdriver

									from selenium.webdriver.common.by import By

									from selenium.webdriver.support.ui import WebDriverWait

									from selenium.webdriver.support import expected_conditions as EC

									from selenium.common.exceptions import TimeoutException

									from bs4 import BeautifulSoup

									from pymongo import MongoClient

									from pymongo.errors import PyMongoError

									# monogdb配置信息

									MONGO_HOST = "localhost"

									MONGO_DATABASE = "douyu"

									MONGO_TABLE = "zhibo"

									client = MongoClient(host=MONGO_HOST)

									db = client[MONGO_DATABASE]

									# PhantomJS 命令行相关配置

									# 参见 http://phantomjs.org/api/command-line.html

									SERVICE_ARGS = ['--disk-cache=true', '--load-images=false']

									# driver = webdriver.Chrome() # 有界面

									driver = webdriver.PhantomJS(service_args=SERVICE_ARGS) # 无界面

									delay = 10

									wait = WebDriverWait(driver, delay)

									driver.maximize_window()

									def get_total_pages():

									  url = 'https://www.douyu.com/directory/all'

									  driver.get(url)

									  pages = int(driver.find_element_by_css_selector(

									    '.shark-pager-dot + .shark-pager-item').text)

									  print("正在获取第1页数据")

									  room_list = get_rooms_by_beautifulsoup()

									  save_to_monogodb(room_list)

									  return pages

									# 根据页码获取指定页数据，并将其保存到数据库中

									def parse_page(page_num):

									  print("正在获取第%d页数据" % page_num)

									  try:

									    page_num_box = wait.until(

									      EC.presence_of_element_located(

									        (By.CSS_SELECTOR, "input.jumptxt")))

									    go_btn = wait.until(EC.element_to_be_clickable(

									      (By.CSS_SELECTOR, 'a.shark-pager-submit')))

									    page_num_box.clear()

									    page_num_box.send_keys(page_num)

									    go_btn.click()

									    # driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

									    # time.sleep(0.1)

									    wait.until(

									      EC.text_to_be_present_in_element(

									        (By.CSS_SELECTOR,

									         '.shark-pager-item.current'),

									        str(page_num)))

									    # 对于By.CLASS_NAME invalid selector: Compound class names not permitted

									    room_list = get_rooms_by_beautifulsoup()

									    save_to_monogodb(room_list)

									  except TimeoutException:

									    print("请求第%d页失败" % page_num)

									    print("尝试重新获取第%d页" % page_num)

									    return parse_page(page_num)

									# 通过bs4解析数据

									def get_rooms_by_beautifulsoup():

									  '''

									  通过bs4库解析数据

									  获取直播间的名称，观看人数,标签，主播名

									  '''

									  wait.until(EC.presence_of_element_located(

									    (By.CSS_SELECTOR, "ul#live-list-contentbox > li")))

									  html = driver.page_source

									  soup = BeautifulSoup(html, 'lxml')

									  rooms = soup.select('ul#live-list-contentbox > li')

									  for room in rooms:

									    room_name = room.find(

									      'h3', attrs={

									        'class': 'ellipsis'}).get_text(

									      strip=True)

									    view_count = room.find('span', class_='dy-num fr').text

									    tag = room.find('span', class_='tag ellipsis').text

									    hostname = room.find('span', class_='dy-name ellipsis fl').text

									    #print("房间名: " + room_name + "\t观看人数: " + view_count + "\t标签: " + tag + "\t主播名: " + hostname)

									    yield {

									      'room_name': room_name,

									      'view_count': view_count,

									      'tag': tag,

									      'hostname': hostname,

									    }

									def save_to_monogodb(room_list):

									  for room in room_list:

									    try:

									      db[MONGO_TABLE].insert(room)  # insert支持插入多条数据

									      print("mongodb插入数据成功:", room)

									    except PyMongoError as e:

									      print("mongodb插入数据失败:", room, e)

									if __name__ == '__main__':

									  try:

									    total_pages = get_total_pages()

									    for page_num in range(2, total_pages + 1):

									      parse_page(page_num)

									  except Exception as e:

									    print("出错了", e)

									  finally: # 确保 浏览器能正常关闭

									    print("共有%d页" % total_pages)

									    driver.close()
希望本文所述对大家Python程序设计有所帮助。
原文链接：http://www.cnblogs.com/hupeng1234/p/7138551.html
Python使用Selenium模块模拟浏览器抓取斗鱼直播间信息示例

相关文章

热门资讯
