1
2
3
4
5
6
7
8
def mm(url):
    try:
        req = requests.get(url, timeout=10)
        req.raise_for_status()  # status_code != 200 抛异常
        req.encoding = req.apparent_encoding
        return req.text
    except Exception:
        return 'wrong'

1
2
3
4
5
6
7
8
9
10
headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'
    }

param = {'query':kw}
response = requests.get(url=url, params=param, headers=headers)

data = {'kw':word}
response = requests.post(url=url, data=data, headers=headers)
dic_obj = response.json()  # 转json

1
2
3
img_data = requests.get(url=url).content  # 二进制内容
with open('./img.jpg','wb') as fp:      # 保存图片
    fp.write(img_data)

1
2
regex = re.compile(r'[1‐9]\d{5}')  # 先编译
match = regex.search('BIT 100081')

1
2
3
4
5
6
7
8
9
10
11
12
13
match = re.search(r'[1-9]\d{5}', 'BIT 100081')  # 查找匹配
print(type(match))
if match:
    print(match.group(0))  # 匹配内容

for m in re.finditer(r'[1-9]\d{5}', 'BIT 100081 aaa111111'):  # 返回第一个匹配的迭代
    if m:
        print(m.group(0))
        
re.findall(r'(\d{4})\w{3}', '3333www4444wwws5555www')
# ['3333', '4444', '5555']
re.findall(r'(\d{4})(\w{3})', '3333www4444wwws5555www')
# [('3333', 'www'), ('4444', 'www'), ('5555', 'www')]

1
2
3
4
5
6
7
from bs4 import BeautifulSoup

page_text = response.text
soup = BeautifulSoup(page_text, 'html.parser')

fp = open('test.html','r',encoding='utf-8')
soup = BeautifulSoup(fp, 'lxml')

1
2
3
4
5
6
7
8
# soup.<tagName> 返回第一个出现的tagName标签
soup.a         # a标签  <a>..</a>
soup.a.name    # 标签的名字
soup.a.attrs   # 标签的属性，字典类型
soup.a.attrs['class']  # 属性的值
soup.a.get('href')     # 属性的值
soup.a['class']        # 属性的值
soup.a.string  # <>..</>之间的内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
soup.body.contents  # 获得body的子节点的列表

for child in soup.body.children:     # 儿子节点
    print(child)

for child in soup.body.descendants:  # 子孙节点
    print(child)

soup.parent          # 父标签 None
soup.title.parent    # 父标签
for parent in soup.a.parents:  # 父辈节点
    if parent is None:
        print(parent)
    else:
        print(parent.name)

soup.a.next_sibling       # 下一个平行节点标签
soup.a.next_sibling.next_sibling
for s in soup.a.next_siblings:
    print(s)

soup.a.previous_sibling   # 上一个平行节点标签
soup.a.previous_sibling.previous_sibling
for s in soup.a.previous_siblings:
    print(s)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# 查找name
soup.find_all('a')  # 查找标签a,返回列表
soup('a')           # 等价写法
soup.find_all(['a', 'b']):        # 传入标签名称列表
soup.find_all(True):              # 返回全部标签
soup.find_all(re.compile('b')):   # 查找标签名称符合正则的标签

# 标签<tag>也可以find_all
soup.body.find_all('a')
soup.body('a')  # 等价写法

# 查找attrs
soup.find_all('p', 'course')   # 不指定属性名，只要属性中带有course即可
soup.find_all(id='link1')      # 指定属性名，查属性id=link1的标签
soup.find_all(id=re.compile('link'))      # 指定属性名，查属性id符合正则的标签
soup.find_all(class_='link1')  # 指定属性名，查属性class=link1的标签

# 参数recursive
soup.find_all('a', recursive=False)  # False表示只查儿子

# 查找string
# 检索<>...</>之间的字符串，返回字符串的列表
soup.find_all(string='Basic Python')
soup(string='Basic Python')  # 等价写法
soup.find_all(string=re.compile('python'))

1
2
soup.select('.tang > ul > li > a')  # >表示的是一个层级
soup.select('.tang > ul a')         # 空格表示的多个层级

1
2
soup.prettify()    # 可以用于BeautifulSoup
soup.a.prettify()  # 也可以用于Tag

1
2
3
4
5
6
7
8
9
10
11
from lxml import etree

# 将本地的html文档中的源码数据加载到etree对象中
tree = etree.parse(filePath)

# 将从互联网上获取的源码数据加载到etree对象中
page_text = response.text
tree = etree.HTML(page_text)

# 返回的是列表
tree.xpath('xpath表达式')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import base64
import json

def base64_api(uname, pwd, img):
    with open(img, 'rb') as f:
        base64_data = base64.b64encode(f.read())
        b64 = base64_data.decode()
    data = {"username": uname, "password": pwd, "image": b64}
    result = json.loads(requests.post("http://api.ttshitu.com/base64", json=data).text)
    if result['success']:
        return result["data"]["result"]
    else:
        return result["message"]

code = base64_api('', '', './img.jpg')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
import requests
from hashlib import md5


class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password =  password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


if __name__ == '__main__':
    # 用户中心>>软件ID 生成一个替换 96001
    chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '96001')
    # 本地图片文件路径 来替换 a.jpg
    im = open('./a.jpg', 'rb').read()
    # 1902 验证码类型 参考 http://www.chaojiying.com/price.html
    print(chaojiying.PostPic(im, 1902))	

1
2
3
4
5
6
7
8
# 创建一个session对象
session = requests.Session()

# 使用session对象进行模拟登录post请求的发送，cookie就会被存储在session中
session.post(url=login_url, headers=headers, data=data)

# 之后使用携带了cookie的session对象发送get请求，就保持在登录状态了
session.get(url=url, headers=headers)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 创建session对象
session = requests.Session()
page = session.get(url=url, headers=headers).text

# xpath解析验证码的路径
tree = etree.HTML(page)
img_path = 'https://so.gushiwen.cn/' + tree.xpath('//*[@id="imgCode"]/@src')[0]

# 保存验证码 这里使用session.get 而不是requests.get 否则验证码会刷新！
img = session.get(url=img_path, headers=headers).content
with open('./img.jpg', 'wb') as fp:
    fp.write(img)
 
# 使用打码平台识别验证码
code = base64_api('', '', './img.jpg')

# 使用验证码进行模拟登陆
data = {'code': code, 'username': '', 'password': ''}
response = session.post(url=post_url, data=data, headers=headers)

1
requests.get(url=url, headers=headers, proxies={"http": 'http://183.166.139.251:9999'})

1
2
3
4
5
6
7
from multiprocessing.dummy import Pool  # 线程池
# from multiprocessing import Pool      # 进程池

pool = Pool(4)        # 4个线程
pool.map(func, list)  # 对list中的每个元素执行func
pool.close()
pool.join()

1
2
3
4
5
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
browser = webdriver.Chrome(chrome_options=chrome_options)

1
2
3
4
firefox_options = FirefoxOptions()
firefox_options.add_argument('--headless')
firefox_options.add_argument('--disable-gpu')
browser = webdriver.Firefox(firefox_options=firefox_options)

1
2
3
4
5
6
7
8
9
from selenium import webdriver

browser = webdriver.Chrome()
# 不设置环境变量的话，需要直接给出驱动的路径
browser = webdriver.Chrome(executable_path='./chromedriver.exe')

browser.get('url')

page_text = browser.page_source

1
2
3
4
5
6
7
8
browser.get('https://www.taobao.com/')
# 标签定位
search_input = browser.find_element_by_id('q')
# 标签交互
search_input.send_keys('Iphone')
# 点击搜索按钮
button = browser.find_element_by_css_selector('.btn-search')
button.click()

1
2
3
4
5
6
7
8
9
browser = webdriver.Chrome()

browser.get('https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')

browser.switch_to.frame('iframe_id')

div = browser.find_element_by_id('div_id')

browser.switch_to.default_content()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
from selenium.webdriver import ActionChains

# 点击操作
action = ActionChains(driver)
click_btn = driver.find_element_by_xpath('')
action.click(click_btn).perform()

# 移动操作
action = ActionChains(driver)
write = driver.find_element_by_xpath('')
action.move_to_element(write).perform()  # 移动到write元素
action.move_by_offset(10, 50).perform()  # 移动到距离当前位置(10,50)的点
action.move_to_element_with_offset(write, 10, -40).perform()  # 移动到距离write元素(10,-40)的点

# 拖拽操作
dragger = driver.find_element_by_id('dragger')  # 被拖拽元素
item1 = driver.find_element_by_xpath('')  # 目标1
item2 = driver.find_element_by_xpath('')  # 目标2
item3 = driver.find_element_by_xpath('')  # 目标3
item4 = driver.find_element_by_xpath('')  # 目标4

action = ActionChains(driver)
action.drag_and_drop(dragger, item1).perform()  # 1.移动dragger到目标1
action.click_and_hold(dragger).release(item2).perform()  # 2.移动dragger到目标2
action.click_and_hold(dragger).move_to_element(item3).release().perform()  # 3.移动dragger到目标3
action.drag_and_drop_by_offset(dragger, 400, 150).perform()  # 4.移动到指定坐标
action.click_and_hold(dragger).move_by_offset(400, 150).release().perform()  # 5.移动到指定坐标

1
2
3
4
5
from PIL import Image
rangle = (5,5,10,10)
i = Image.open('./page.png')  # 打开图片
frame = i.crop(rangle)        # 截取一个矩形区域
frame.save('./code.png')      # 保存图片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
from selenium.webdriver import ActionChains
from PIL import Image
import time
from selenium import webdriver
from Chaojiying import Chaojiying_Client   # 超级鹰打码平台接口

browser = webdriver.Chrome()

# 浏览器全屏
browser.maximize_window()

browser.get('https://kyfw.12306.cn/otn/login/init')

# 页面截屏
browser.save_screenshot('./page.png')

# 找到验证码标签
img = browser.find_element_by_xpath('//*[@id="loginForm"]/div/ul[2]/li[4]/div/div/div[3]/img')
location = img.location
size = img.size

# 从页面截屏中截取验证码区域
rangle = (int(location['x']), int(location['y']),
          int(location['x'] + size['width']), int(location['y'] + size['height']))
i = Image.open('./page.png')
frame = i.crop(rangle)
frame.save('./code.png')

# 验证码识别
chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '软件ID')
im = open('./code.png', 'rb').read()
result = chaojiying.PostPic(im, 9004)['pic_str']

all_list = []  # 存储即将被点击的点的坐标  [[x1,y1],[x2,y2]]
if '|' in result:
    list_1 = result.split('|')
    count_1 = len(list_1)
    for i in range(count_1):
        xy_list = []
        x = int(list_1[i].split(',')[0])
        y = int(list_1[i].split(',')[1])
        xy_list.append(x)
        xy_list.append(y)
        all_list.append(xy_list)
else:
    x = int(result.split(',')[0])
    y = int(result.split(',')[1])
    xy_list = []
    xy_list.append(x)
    xy_list.append(y)
    all_list.append(xy_list)
print(all_list)

# 输入账号密码
browser.find_element_by_xpath('//*[@id="username"]').send_keys('')
time.sleep(2)
browser.find_element_by_xpath('//*[@id="password"]').send_keys('')
time.sleep(2)

# 使用动作链对每一个列表元素对应的x,y指定的位置进行点击操作
for l in all_list:
    # 相对于验证码标签<img>进行偏移，确定位置后点击
    ActionChains(browser).move_to_element_with_offset(img, l[0], l[1]).click().perform()
    time.sleep(0.5)

# 点击登录按钮
browser.find_element_by_xpath('//*[@id="loginSub"]').click()
time.sleep(10)

browser.quit()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from selenium import webdriver

# 实现无可视化界面
from selenium.webdriver.chrome.options import Options

# 实现规避检测
from selenium.webdriver import ChromeOptions

# 实现无可视化界面
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

# 实现规避检测
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])

browser = webdriver.Chrome(chrome_options=chrome_options, options=option)


from selenium.webdriver import FirefoxOptions
# 实现无可视化界面
firefox_options = FirefoxOptions()
firefox_options.add_argument('--headless')
firefox_options.add_argument('--disable-gpu')
browser = webdriver.Firefox(firefox_options=firefox_options)

1
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'

1
ROBOTSTXT_OBEY = False

1
LOG_LEVEL = 'ERROR'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
import scrapy


class DemoSpider(scrapy.Spider):
    # 爬虫名
    name = "demo"
    # 允许访问的域名，用于限定start_urls中哪些可用，基本不用这个属性
    # allowed_domains = ["python123.io"]
    # 起始url列表
    # start_urls = ['https://python123.io/ws/demo.html']

    # 此函数等价于start_urls = ['https://python123.io/ws/demo.html']
    def start_requests(self):
        urls = [
            'https://python123.io/ws/demo.html'
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    # 解析函数 对于start_urls中的爬取结果进行解析
    def parse(self, response):
        # 数据解析处理Response对象，并封装成Item()对象
        yield item

1
2
3
4
5
6
7
8
div_list = response.xpath('//div')
for div in div_list:
    div.xpath('./***').extract_first()
    div.xpath('./***').extract()
    div.xpath('./***')[0].extract()

response.xpath('/***').extract()
response.xpath('//**')[0].extract()

1
2
3
4
5
6
7
8
9
10
11
12
import scrapy


class MyfirstpjtItem(scrapy.Item):
    # 定义Item对象的内容
    urlname = scrapy.Field()
    urlkey = scrapy.Field()

# Item()使用方法
item = MyfirstpjtItem()
item['urlname'] = ''
item['urlkey'] = ''

1
2
3
4
ITEM_PIPELINES = {
    'myfirstpjt.pipelines.MyfirstpjtPipeline': 300,
}
# 300为优先级，数值越小优先级越大

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
import json
import codecs


class MyfirstpjtPipeline(object):
    # 开始时调用一次
    def open_spider(self,spider):
        self.file = codecs.open('mydata.json', 'wb', encoding='utf-8')

    # 专门用来处理item类型对象
    # 该方法每接收到一个item就会被调用一次
    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.file.write(line)
        return item   # ！！！一定要return item

    # 结束时调用一次
    def close_spider(self, spider):
        self.file.close()

1
2
3
4
5
6
7
def parse(self, response):
    if self.page_num <= 11:
        new_url = format(self.url%self.page_num)
        self.page_num += 1
        # 手动请求发送	
        # callback回调函数是专门用作于数据解析，可以指定为另外编写的函数，也可以用parse()
        yield scrapy.Request(url=new_url,callback=self.parse)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
class BossSpider(scrapy.Spider):
    name = 'boss'
    start_urls = ['https://www.zhipin.com/job_detail/?query=python&city=101010100&industry=&position=']

    url = 'https://www.zhipin.com/c101010100/?query=python&page=%d'
    page_num = 2

    # 回调函数接受item
    def parse_detail(self, response):
        item = response.meta['item']
        job_desc = response.xpath('').extract()
        item['job_desc'] = job_desc
        # 在这个解析函数中yield item
        yield item

    # 解析首页中的岗位名称
    def parse(self, response):
        li_list = response.xpath('//*[@id="main"]/div/div[3]/ul/li')
        for li in li_list:
            item = BossproItem()
            job_name = li.xpath('').extract_first()
            item['job_name'] = job_name
            detail_url = '...'
            # 对详情页手动发起请求 获取详情页的页面源码数据
            # 请求传参：meta={}，可以将meta字典传递给请求对应的回调函数
            yield scrapy.Request(url=detail_url, callback=self.parse_detail, meta={'item':item})
        # 分页操作
        if self.page_num <= 3:
            new_url = format(self.url%self.page_num)
            self.page_num += 1
            # 不同页码的页面都使用parse()函数
            yield scrapy.Request(url=new_url, callback=self.parse)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# pipelines.py
from scrapy.pipelines.images import ImagesPipeline
import scrapy


class PicpjtPipeline(ImagesPipeline):

    # 根据图片地址进行图片数据的请求
    def get_media_requests(self, item, info):
        yield scrapy.Request(item['src'])

    # 指定图片存储的路径，返回文件名即可。路径在setting.py中指定
    def file_path(self, request, response=None, info=None):
        imgName = request.url.split('/')[-1]
        return imgName

    # 返回给下一个即将被执行的管道类
    def item_completed(self, results, item, info):
        return item


# settings.py
ITEM_PIPELINES = {
   'picpjt.pipelines.PicpjtPipeline': 300,
}
IMAGES_STORE = './imgs'   

# items.py
class PicpjtItem(scrapy.Item):
    src = scrapy.Field()

# spiderName.py
# 解析图片src，返回Item()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# middlewares.py的MiddlepjtDownloaderMiddleware类中
# 拦截所有请求，进行UA伪装
def process_request(self, request, spider):
    # UA伪装  user_agent_list中随机挑选一个
    request.headers['User-Agent'] = random.choice(self.user_agent_list)
    return None

# 拦截发生异常的请求，进行IP代理
# 必须将修改后的Request返回：return request
# 根据url选择http或https，从IP池中随机选择代理
def process_exception(self, request, exception, spider):
    if request.url.split(':')[0] == 'http':
        # IP代理
        request.meta['proxy'] = 'http://' + random.choice(self.PROXY_http)
    else:
        request.meta['proxy'] = 'https://' + random.choice(self.PROXY_https)
    # 将修正之后的请求对象进行重新发送！！
    return request

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# middlewares.py的MiddlepjtDownloaderMiddleware类中
from scrapy.http import HtmlResponse
from time import sleep

def process_response(self, request, response, spider):
    # 获取了在爬虫类中定义的浏览器对象
    browser = spider.browser
    # 根据url挑选出指定的响应对象进行篡改(只篡改想要篡改的响应)
    if request.url in spider.models_urls:
        browser.get(request.url)  # 打开页面
        sleep(3)
        page_text = browser.page_source  # 动态加载的数据
        # 对响应进行篡改，修改其内容body=page_text
        new_response = HtmlResponse(url=request.url, body=page_text, encoding='utf-8', request=request)
        return new_response
    else:
        # 其他请求对应的响应对象直接返回
        return response
    
# SpiderName.py中
# 需要实例化selenium.webdriver 浏览器对象
from selenium import webdriver

# 爬虫初始化
def __init__(self):
    # 实例化一个浏览器对象
    self.browser = webdriver.Chrome()

# 爬虫关闭时
def closed(self, spider):
    # 关闭浏览器对象
    self.browser.quit()

# 解析函数如parse()就跟一般的没有区别，因为动态加载的内容直接从中间件返回回来了

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from mycrawlpjt.items import MycrawlpjtItem


class MycrawlSpider(CrawlSpider):
    name = 'mycrawl'
    allowed_domains = ['sohu.com']
    start_urls = ['http://news.sohu.com/']

    rules = (
        # http://www.sohu.com/a/328591875_267106
        # 提取链接的规则
        Rule(  # 规则解析器
            LinkExtractor(   # 链接提取器
                allow='http://www\\.sohu\\.com/a/.*',  # 正则
                allow_domains='sohu.com',
            ),
            callback='parse_item',  # 每调用一次就解析一次链接提取器提取到链接对应的页面内容
            follow=True),  # 对链接提取器提取到的链接的页面继续提取链接
    )

    def parse_item(self, response):  # 解析已经提取出来的链接
        item = MycrawlpjtItem()
        item['name'] = response.xpath("/html/head/title/text()").extract()
        item['link'] = response.xpath("//link[@rel='canonical']/@href").extract()
        return item

1
2
3
4
5
6
start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1&page=1']

rules = (
        Rule(LinkExtractor(allow='/political/index/politicsNewest\?id=1&page=\\d+'), callback='parse_item', follow=True),
    )
# 因为这个页面中的其他页面的链接就是不完整的，不是http开头，所以allow的正则才是这样的！！

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
class SunSpider(CrawlSpider):
    name = 'sun'
    start_urls = ['']

    # 链接提取器：根据指定规则（allow="正则"）进行指定链接的提取
    link = LinkExtractor(allow=r'')
    link_detail = LinkExtractor(allow=r'')
    rules = (
        # 规则解析器：使用多个Rule
        Rule(link, callback='parse_item', follow=True),
        Rule(link_detail,callback='parse_detail')
    )

    # 结合不同的数据解析函数
    def parse_item(self, response):
        item = SunproItem()
        ...
        yield item

    def parse_detail(self,response):
        item = DetailItem()
        ...
        yield item

1
2
3
ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 400
}

1
2
3
4
5
6
7
# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 使用scrapy-redis组件的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。
# 如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True

1
2
REDIS_HOST = '127.0.0.1' #redis远程服务器的ip（修改）
REDIS_PORT = 6379

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
from redis import Redis


class MovieSpider(CrawlSpider):
	conn = Redis(host='127.0.0.1', port=6379)
    
    def parse_item(self, response):
        li_list = response.xpath('/html/body/div[1]/div/div/div/div[2]/ul/li')
        for li in li_list:
            # 获取详情页的url
            detail_url = li.xpath('./div/a/@href').extract_first()
            # 将详情页的url存入redis的set中
            ex = self.conn.sadd('urls', detail_url)
            if ex == 1:
                print('该url没有被爬取过，可以进行数据的爬取')
                yield scrapy.Request(url=detail_url, callback=self.parse_detail)
            else:
                print('数据还没有更新，暂无新数据可爬取！')
      
    def parse_detail(self, response):
        yield item
                
class MoviepjtPipeline(object):
    # 开始时调用一次
    def open_spider(self,spider):
        # 追加模式 避免覆盖
        self.file = codecs.open('mydata.json', 'ab', encoding='utf-8')

    # 专门用来处理item类型对象
    # 该方法每接收到一个item就会被调用一次
    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.file.write(line)
        return item

    # 结束时调用一次
    def close_spider(self, spider):
        self.file.close()