Python的几个爬虫代码整理（网易云、微信、淘宝、今日头条）

#!/usr/bin/env python
# encoding=utf-8
from __future__ import print_function

import os
import requests
import re
import time
import xml.dom.minidom
import json
import sys
import math
import subprocess
import ssl
import threading
import urllib, urllib2

DEBUG = False

MAX_GROUP_NUM = 2  # 每组人数
INTERFACE_CALLING_INTERVAL = 5  # 接口调用时间间隔, 间隔太短容易出现"操作太频繁", 会被限制操作半小时左右
MAX_PROGRESS_LEN = 50

QRImagePath = os.path.join(os.getcwd(), ‘qrcode.jpg‘)

tip = 0
uuid = ‘‘

base_uri = ‘‘
redirect_uri = ‘‘
push_uri = ‘‘

skey = ‘‘
wxsid = ‘‘
wxuin = ‘‘
pass_ticket = ‘‘
deviceId = ‘e000000000000000‘

BaseRequest = {}

ContactList = []
My = []
SyncKey = []

try:
    xrange
    range = xrange
except:
    # python 3
    pass


def responseState(func, BaseResponse):
    ErrMsg = BaseResponse[‘ErrMsg‘]
    Ret = BaseResponse[‘Ret‘]
    if DEBUG or Ret != 0:
        print(‘func: %s, Ret: %d, ErrMsg: %s‘ % (func, Ret, ErrMsg))

    if Ret != 0:
        return False

    return True


def getUUID():
    global uuid

    url = ‘https://login.weixin.qq.com/jslogin‘
    params = {
        ‘appid‘: ‘wx782c26e4c19acffb‘,
        ‘fun‘: ‘new‘,
        ‘lang‘: ‘zh_CN‘,
        ‘_‘: int(time.time()),
    }

    r = myRequests.get(url=url, params=params)
    r.encoding = ‘utf-8‘
    data = r.text

    # print(data)

    # window.QRLogin.code = 200; window.QRLogin.uuid = "oZwt_bFfRg==";
    regx = r‘window.QRLogin.code = (\d+); window.QRLogin.uuid = "(\S+?)"‘
    pm = re.search(regx, data)

    code = pm.group(1)
    uuid = pm.group(2)

    if code == ‘200‘:
        return True

    return False


def showQRImage():
    global tip

    url = ‘https://login.weixin.qq.com/qrcode/‘ + uuid
    params = {
        ‘t‘: ‘webwx‘,
        ‘_‘: int(time.time()),
    }

    r = myRequests.get(url=url, params=params)
    tip = 1

    f = open(QRImagePath, ‘wb+‘)
    f.write(r.content)
    f.close()
    time.sleep(1)

    if sys.platform.find(‘darwin‘) >= 0:
        subprocess.call([‘open‘, QRImagePath])
    else:
        os.startfile(QRImagePath)

    print(‘请使用微信扫描二维码以登录‘)


def waitForLogin():
    global tip, base_uri, redirect_uri, push_uri

    url = ‘https://login.weixin.qq.com/cgi-bin/mmwebwx-bin/login?tip=%s&uuid=%s&_=%s‘ % (
        tip, uuid, int(time.time()))

    r = myRequests.get(url=url)
    r.encoding = ‘utf-8‘
    data = r.text

    # print(data)

    # window.code=500;
    regx = r‘window.code=(\d+);‘
    pm = re.search(regx, data)

    code = pm.group(1)

    if code == ‘201‘:  # 已扫描
        print(‘成功扫描,请在手机上点击确认以登录‘)
        tip = 0
    elif code == ‘200‘:  # 已登录
        print(‘正在登录...‘)
        regx = r‘window.redirect_uri="(\S+?)";‘
        pm = re.search(regx, data)
        redirect_uri = pm.group(1) + ‘&fun=new‘
        base_uri = redirect_uri[:redirect_uri.rfind(‘/‘)]

        # push_uri与base_uri对应关系(排名分先后)(就是这么奇葩..)
        services = [
            (‘wx2.qq.com‘, ‘webpush2.weixin.qq.com‘),
            (‘qq.com‘, ‘webpush.weixin.qq.com‘),
            (‘web1.wechat.com‘, ‘webpush1.wechat.com‘),
            (‘web2.wechat.com‘, ‘webpush2.wechat.com‘),
            (‘wechat.com‘, ‘webpush.wechat.com‘),
            (‘web1.wechatapp.com‘, ‘webpush1.wechatapp.com‘),
        ]
        push_uri = base_uri
        for (searchUrl, pushUrl) in services:
            if base_uri.find(searchUrl) >= 0:
                push_uri = ‘https://%s/cgi-bin/mmwebwx-bin‘ % pushUrl
                break

        # closeQRImage
        if sys.platform.find(‘darwin‘) >= 0:  # for OSX with Preview
            os.system("osascript -e ‘quit app \"Preview\"‘")
    elif code == ‘408‘:  # 超时
        pass
    # elif code == ‘400‘ or code == ‘500‘:

    return code


def login():
    global skey, wxsid, wxuin, pass_ticket, BaseRequest

    r = myRequests.get(url=redirect_uri)
    r.encoding = ‘utf-8‘
    data = r.text

    # print(data)

    doc = xml.dom.minidom.parseString(data)
    root = doc.documentElement

    for node in root.childNodes:
        if node.nodeName == ‘skey‘:
            skey = node.childNodes[0].data
        elif node.nodeName == ‘wxsid‘:
            wxsid = node.childNodes[0].data
        elif node.nodeName == ‘wxuin‘:
            wxuin = node.childNodes[0].data
        elif node.nodeName == ‘pass_ticket‘:
            pass_ticket = node.childNodes[0].data

    # print(‘skey: %s, wxsid: %s, wxuin: %s, pass_ticket: %s‘ % (skey, wxsid,
    # wxuin, pass_ticket))

    if not all((skey, wxsid, wxuin, pass_ticket)):
        return False

    BaseRequest = {
        ‘Uin‘: int(wxuin),
        ‘Sid‘: wxsid,
        ‘Skey‘: skey,
        ‘DeviceID‘: deviceId,
    }

    return True


def webwxinit():
    url = (base_uri +
           ‘/webwxinit?pass_ticket=%s&skey=%s&r=%s‘ % (
               pass_ticket, skey, int(time.time())))
    params = {‘BaseRequest‘: BaseRequest}
    headers = {‘content-type‘: ‘application/json; charset=UTF-8‘}

    r = myRequests.post(url=url, data=json.dumps(params), headers=headers)
    r.encoding = ‘utf-8‘
    data = r.json()

    if DEBUG:
        f = open(os.path.join(os.getcwd(), ‘webwxinit.json‘), ‘wb‘)
        f.write(r.content)
        f.close()

    # print(data)

    global ContactList, My, SyncKey
    dic = data
    ContactList = dic[‘ContactList‘]
    My = dic[‘User‘]
    SyncKey = dic[‘SyncKey‘]

    state = responseState(‘webwxinit‘, dic[‘BaseResponse‘])
    return state


def webwxgetcontact():
    url = (base_uri +
           ‘/webwxgetcontact?pass_ticket=%s&skey=%s&r=%s‘ % (
               pass_ticket, skey, int(time.time())))
    headers = {‘content-type‘: ‘application/json; charset=UTF-8‘}

    r = myRequests.post(url=url, headers=headers)
    r.encoding = ‘utf-8‘
    data = r.json()

    if DEBUG:
        f = open(os.path.join(os.getcwd(), ‘webwxgetcontact.json‘), ‘wb‘)
        f.write(r.content)
        f.close()

    dic = data
    MemberList = dic[‘MemberList‘]

    # 倒序遍历,不然删除的时候出问题..
    SpecialUsers = ["newsapp", "fmessage", "filehelper", "weibo", "qqmail", "tmessage", "qmessage", "qqsync",
                    "floatbottle", "lbsapp", "shakeapp", "medianote", "qqfriend", "readerapp", "blogapp", "facebookapp",
                    "masssendapp",
                    "meishiapp", "feedsapp", "voip", "blogappweixin", "weixin", "brandsessionholder", "weixinreminder",
                    "wxid_novlwrv3lqwv11", "gh_22b87fa7cb3c", "officialaccounts", "notification_messages", "wxitil",
                    "userexperience_alarm"]
    for i in range(len(MemberList) - 1, -1, -1):
        Member = MemberList[i]
        if Member[‘VerifyFlag‘] & 8 != 0:  # 公众号/服务号
            MemberList.remove(Member)
        elif Member[‘UserName‘] in SpecialUsers:  # 特殊账号
            MemberList.remove(Member)
        elif Member[‘UserName‘].find(‘@@‘) != -1:  # 群聊
            MemberList.remove(Member)
        elif Member[‘UserName‘] == My[‘UserName‘]:  # 自己
            MemberList.remove(Member)

    return MemberList


def syncKey():
    SyncKeyItems = [‘%s_%s‘ % (item[‘Key‘], item[‘Val‘])
                    for item in SyncKey[‘List‘]]
    SyncKeyStr = ‘|‘.join(SyncKeyItems)
    return SyncKeyStr


def syncCheck():
    url = push_uri + ‘/synccheck?‘
    params = {
        ‘skey‘: BaseRequest[‘Skey‘],
        ‘sid‘: BaseRequest[‘Sid‘],
        ‘uin‘: BaseRequest[‘Uin‘],
        ‘deviceId‘: BaseRequest[‘DeviceID‘],
        ‘synckey‘: syncKey(),
        ‘r‘: int(time.time()),
    }

    r = myRequests.get(url=url, params=params)
    r.encoding = ‘utf-8‘
    data = r.text

    # print(data)

    # window.synccheck={retcode:"0",selector:"2"}
    regx = r‘window.synccheck={retcode:"(\d+)",selector:"(\d+)"}‘
    pm = re.search(regx, data)

    retcode = pm.group(1)
    selector = pm.group(2)

    return selector


def webwxsync():
    global SyncKey

    url = base_uri + ‘/webwxsync?lang=zh_CN&skey=%s&sid=%s&pass_ticket=%s‘ % (
        BaseRequest[‘Skey‘], BaseRequest[‘Sid‘], urllib.quote_plus(pass_ticket))
    params = {
        ‘BaseRequest‘: BaseRequest,
        ‘SyncKey‘: SyncKey,
        ‘rr‘: ~int(time.time()),
    }
    headers = {‘content-type‘: ‘application/json; charset=UTF-8‘}

    r = myRequests.post(url=url, data=json.dumps(params))
    r.encoding = ‘utf-8‘
    data = r.json()

    # print(data)

    dic = data
    SyncKey = dic[‘SyncKey‘]

    state = responseState(‘webwxsync‘, dic[‘BaseResponse‘])
    return state


def heartBeatLoop():
    while True:
        selector = syncCheck()
        if selector != ‘0‘:
            webwxsync()
        time.sleep(1)


def main():
    global myRequests

    if hasattr(ssl, ‘_create_unverified_context‘):
        ssl._create_default_https_context = ssl._create_unverified_context

    headers = {
        ‘User-agent‘: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36‘}
    myRequests = requests.Session()
    myRequests.headers.update(headers)

    if not getUUID():
        print(‘获取uuid失败‘)
        return

    print(‘正在获取二维码图片...‘)
    showQRImage()

    while waitForLogin() != ‘200‘:
        pass

    os.remove(QRImagePath)

    if not login():
        print(‘登录失败‘)
        return

    if not webwxinit():
        print(‘初始化失败‘)
        return

    MemberList = webwxgetcontact()

    threading.Thread(target=heartBeatLoop)

    MemberCount = len(MemberList)
    print(‘通讯录共%s位好友‘ % MemberCount)

    d = {}
    imageIndex = 0
    for Member in MemberList:
        imageIndex = imageIndex + 1
        # name = ‘C:\\Users\\Public\\Pictures\\‘ + str(imageIndex) + ‘.jpg‘
        # imageUrl = ‘http://wx2.qq.com‘ + Member[‘HeadImgUrl‘]
        # r = myRequests.get(url=imageUrl, headers=headers)
        # imageContent = (r.content)
        # fileImage = open(name, ‘wb‘)
        # fileImage.write(imageContent)
        # fileImage.close()
        # print(‘正在下载第：‘ + str(imageIndex) + ‘位好友头像‘)
        d[Member[‘UserName‘]] = (Member[‘NickName‘], Member[‘RemarkName‘])
        city = Member[‘City‘]
        city = ‘nocity‘ if city == ‘‘ else  city
        name = Member[‘NickName‘]
        name = ‘noname‘ if name == ‘‘ else  name
        sign = Member[‘Signature‘]
        sign = ‘nosign‘ if sign == ‘‘ else  sign
        remark = Member[‘RemarkName‘]
        remark = ‘noremark‘ if remark == ‘‘ else remark
        alias = Member[‘Alias‘]
        alias = ‘noalias‘ if alias == ‘‘ else alias
        nick = Member[‘NickName‘]
        nick = ‘nonick‘ if nick == ‘‘ else nick
        print(name, ‘|||‘, city, ‘|||‘, Member[‘Sex‘], ‘|||‘, Member[‘StarFriend‘], ‘|||‘, sign,
              ‘|||‘, remark, ‘|||‘, alias, ‘|||‘, nick)



if __name__ == ‘__main__‘:
    main()
    print(‘回车键退出...‘)
    input()

import re
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
from taobao.config import *
import pymongo

client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)
wait = WebDriverWait(browser, 10)
#设置窗口大小，以免默认的相对较小的窗口影响操作
browser.set_window_size(1400, 900)

def search():
    print(‘正在搜索‘)
    try:
        browser.get(‘https://www.taobao.com‘)
        # http://selenium-python.readthedocs.io/waits.html#explicit-waits
        # 输入框
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, ‘#q‘))
        )
        #搜索提交按钮
        submit = wait.until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, ‘#J_TSearchForm > div.search-button > button‘)))

        input.send_keys(KEYWORD)
        submit.click()

        #等待搜索内容加载，表示为共x页
        total = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, ‘#mainsrp-pager > div > div > div > div.total‘)))
        #total中就是总共有多少页
        get_products()
        return total.text
    except TimeoutException:
        # wait在网速过慢的时候会出现超时错误，所以我们递归调用重新请求
        return search()


def next_page(page_number):
    #进入下一页有多种方式，比如点击xx页，比如点击下一页按钮
    #比如在输入框中设置页码，点击确定
    #我们采用最后一种，因为第一章容易错乱
    #而最后一种相比于第二种方便我们在出错的时候递归
    print(‘正在翻页‘, page_number)
    try:
        #输入页码框
        input = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, ‘#mainsrp-pager > div > div > div > div.form > input‘))
        )
        #提交按钮
        submit = wait.until(EC.element_to_be_clickable(
            (By.CSS_SELECTOR, ‘#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit‘)))
        #清楚原先的页码
        input.clear()
        input.send_keys(page_number)
        submit.click()
        #判断翻页是否成功（判断条件是高亮的当前代码）
        wait.until(EC.text_to_be_present_in_element(
            (By.CSS_SELECTOR, ‘#mainsrp-pager > div > div > div > ul > li.item.active > span‘), str(page_number)))
        get_products()
    except TimeoutException:
        #出错则递归，重新执行
        next_page(page_number)

#解析
def get_products():
    #判断所有的宝贝信息是否传递成功
    wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ‘#mainsrp-itemlist .items .item‘)))
    html = browser.page_source
    doc = pq(html)
    items = doc(‘#mainsrp-itemlist .items .item‘).items()
    for item in items:
        product = {
            ‘image‘: item.find(‘.pic .img‘).attr(‘src‘),
            ‘price‘: item.find(‘.price‘).text(),
            ‘deal‘: item.find(‘.deal-cnt‘).text()[:-3],
            ‘title‘: item.find(‘.title‘).text(),
            ‘shop‘: item.find(‘.shop‘).text(),
            ‘location‘: item.find(‘.location‘).text()
        }
        print(product)
        save_to_mongo(product)

#保存
def save_to_mongo(result):
    try:
        if db[MONGO_TABLE].insert(result):
            print(‘存储到MONGODB成功‘, result)
    except Exception:
        print(‘存储到MONGODB失败‘, result)


def main():
    try:
        total = search()
        #提取总页数
        total = int(re.compile(‘(\d+)‘).search(total).group(1))
        for i in range(2, total + 1):
            #从第二页才开始需要点击下一页
            next_page(i)
    except Exception:
        print(‘出错啦‘)
    finally:
        browser.close()

if __name__ == ‘__main__‘:
    main()

MONGO_URL = ‘localhost‘
MONGO_DB = ‘taobao‘
MONGO_TABLE = ‘product‘

# http://phantomjs.org/api/command-line.html
#第一个参数是不加载图片，使程序运行更快
#第二个的开启缓存
SERVICE_ARGS = [‘--load-images=false‘, ‘--disk-cache=true‘]

KEYWORD = ‘汉服‘

import json
import os
from urllib.parse import urlencode
import pymongo
import requests
from bs4 import BeautifulSoup
from requests.exceptions import ConnectionError
import re
from multiprocessing import Pool
from hashlib import md5
from json.decoder import JSONDecodeError
from spider_basic.config import *

#
client = pymongo.MongoClient(MONGO_URL, connect=False)
db = client[MONGO_DB]


#获取主页，offse是ajax动态加载的偏移量，keyword是搜索的关键字
def get_page_index(offset, keyword):
    data = {
        ‘autoload‘: ‘true‘,
        ‘count‘: 20,
        ‘cur_tab‘: 3,
        ‘format‘: ‘json‘,
        ‘keyword‘: keyword,
        ‘offset‘: offset,
    }
    params = urlencode(data)
    base = ‘http://www.toutiao.com/search_content/‘
    url = base + ‘?‘ + params
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        print(‘Error occurred‘)
        return None

#下载
def download_image(url):
    print(‘Downloading‘, url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            save_image(response.content)
        return None
    except ConnectionError:
        return None

#保存
def save_image(content):
    # 文件名
    # 路径 名称 后缀
    # 名称md5 这是为了当我们运行一次程序出错时，第二次不再保存重复图片
    file_path = ‘{0}/{1}/{2}.{3}‘.format(os.getcwd(),‘pic‘,md5(content).hexdigest(), ‘jpg‘)
    print(file_path)
    # 如果文件不存在则保存
    # content是二进制形式的网页内容
    if not os.path.exists(file_path):
        with open(file_path, ‘wb‘) as f:
            f.write(content)
            f.close()

#解析搜索后返回的网页（就是图集XHR中的那些）
def parse_page_index(text):
    try:
        #转化为JSON
        data = json.loads(text)
        #如果JSON数据中含有data这个键
        if data and ‘data‘ in data.keys():
            for item in data.get(‘data‘):
                yield item.get(‘article_url‘)
    except JSONDecodeError:
        pass

#获取详情页
def get_page_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        print(‘Error occurred‘)
        return None

#解析详情页
def parse_page_detail(html, url):
    soup = BeautifulSoup(html, ‘lxml‘)
    #获取标题
    result = soup.select(‘title‘)
    title = result[0].get_text() if result else ‘‘
    #获取图片地址（定义正则规则，并search）
    images_pattern = re.compile(‘var gallery = (.*?);‘, re.S)
    result = re.search(images_pattern, html)
    #如果非空
    if result:
        #转化为JSON格式
        data = json.loads(result.group(1))
        #如果sub_images这个键的值对应了各种url
        if data and ‘sub_images‘ in data.keys():
            sub_images = data.get(‘sub_images‘)
            images = [item.get(‘url‘) for item in sub_images]
            #下载图片
            for image in images: download_image(image)
            return {
                ‘title‘: title,
                ‘url‘: url,
                ‘images‘: images
            }

#存储到数据库
def save_to_mongo(result):
    if db[MONGO_TABLE].insert(result):
        print(‘Successfully Saved to Mongo‘, result)
        return True
    return False


def main(offset):
    text = get_page_index(offset, KEYWORD)
    #生成器返回图集中每一个图集的URL的遍历器
    urls = parse_page_index(text)
    #遍历每一个图集，获取详情页信息
    for url in urls:
        #获取详情页
        html = get_page_detail(url)
        #解析详情页
        result = parse_page_detail(html, url)
        #保存到MongoDB
        if result: save_to_mongo(result)


# if __name__ == ‘__main__‘:
#     main(60)

# 注意，如果不在if __name__ == ‘__main__‘:中运行的话，会爆出一堆多线程的错误
if __name__ == ‘__main__‘:
    pool = Pool()
    groups = ([x * 20 for x in range(GROUP_START, GROUP_END + 1)])
    pool.map(main, groups)
    pool.close()
    pool.join()

MONGO_URL = ‘localhost‘
MONGO_DB = ‘toutiao‘
MONGO_TABLE = ‘toutiao‘

GROUP_START = 1
GROUP_END = 20
KEYWORD=‘萌宠‘

附件列表