2回答

bobby

2018-05-21

已采纳

302了 可以先用selenium模拟登录 然后再重新抓取

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import time
import pickle
import datetime
import sys
import io
class LagouSpider(CrawlSpider):
    name = 'lagou_sel'
    allowed_domains = ['www.lagou.com']
    start_urls = ['https://www.lagou.com/']
    headers={
        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding":"gzip, deflate, br",
        "Accept-Language":"zh-CN,zh;q=0.8",
        "Connection":"keep-alive",
        "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
         "Referer":'https://www.lagou.com',
         'Connection': 'keep-alive',
         "HOST": "www.lagou.com"
        }
    custom_settings = {
        "COOKIES_ENABLED": True
    }
    rules = (
        Rule(LinkExtractor(allow=r'gongsi/j/\d+.html'), follow=True),
        Rule(LinkExtractor(allow=r'zhaopin/.*'), follow=True),
        Rule(LinkExtractor(allow=r'jobs/\d+.html'), callback='parse_job', follow=True),
    )
    def parse_item(self, response):
        pass
    def start_requests(self):
        from selenium import webdriver
        sys.stdout=io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')
        chrome_opt=webdriver.ChromeOptions()
        prefs={"profile.managed_default_content_settings.images":2}
        chrome_opt.add_experimental_option("prefs",prefs)
        browser = webdriver.Chrome(executable_path="E:/tmp/chromedriver.exe",chrome_options=chrome_opt)
        browser.get("https://passport.lagou.com/login/login.html?service=https%3a%2f%2fwww.lagou.com%2f")
        browser.find_elements_by_css_selector(".input.input_white")[0].send_keys("xxx")
        browser.find_elements_by_css_selector(".input.input_white")[1].send_keys("xx")
        # browser.find_element_by_xpath("/html/body/section/div[1]/div[2]/form/div[2]/input").send_keys(password)
        browser.find_element_by_css_selector(".btn.btn_green.btn_active.btn_block.btn_lg").click()
        time.sleep(10)
        Cookies = browser.get_cookies()
        cookie_dict={}
        for cookie in Cookies:
            f=open('H:/慕课网课程/python爬虫/课程源码最终版/ArticleSpider/cookies123'+cookie['name']+'.lagou','wb')
            pickle.dump(cookie,f)
            f.close()
            cookie_dict[cookie['name']]=cookie['value']
        browser.close()
        return [scrapy.Request(url=self.start_urls[0], dont_filter=True, cookies=cookie_dict)]

2

6

bobby

回复

蓝莓圣代

拉勾如果重定向了 说明直接通过ip限制了访问，你可以重启一下路由切换ip或者在获取到cookie之后 用scrapy的request代理ip访问，但是如果拉勾将ip和cookie绑定了的话 就只有用当前的ip访问了，同时限制一下抓取速度。

2018-08-15

共6条回复

qq_AGGRESSIVE_0

2018-05-20

我也遇到了，可以通过在parse_job()中添加print()，验证进入了方法，

0

0