AttributeError: 'dict' object has no attribute 'query'

来源:6-20 保存数据到mysql中 -2

兰陵小生

2019-12-10

在插入数据到数据库中,出现如下错误:
图片描述
图片描述
我的items中代码为:

class ZhihuQuestionItem(scrapy.Item):
    title = scrapy.Field()
    url = scrapy.Field()
    topics = scrapy.Field()
    zhihu_id = scrapy.Field()
    content = scrapy.Field()
    answer_num = scrapy.Field()
    comments_num = scrapy.Field()
    watch_user_num = scrapy.Field()
    click_num = scrapy.Field()
    crawl_time = scrapy.Field()
    def get_sql(self):
        insert_sql = '''
                        insert into zhihu_question(title,url,topics,zhihu_id,
                        content,answer_num,comments_num,watch_user_num,click_num,crawl_time)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) 
                        ON DUPLICATE KEY 
                        UPDATE answer_num=VALUES(answer_num),
                        comments_num=VALUES(comments_num),
                        watch_user_num=VALUES(watch_user_num),
                        crawl_time=VALUES(crawl_time),
                        click_num=VALUES(click_num)

                        '''
        zhihu_id = "".join(self["zhihu_id"])
        topics = ",".join(self['topics'])
        url = self["url"][0]
        title = ",".join(self['title'])
        content = ",".join(self['content'])
        answer_num = extract_num(",".join(self['answer_num']))
        comments_num = extract_num(",".join(self['comments_num']))
        watch_user_num = extract_num(",".join(self['watch_user_num']))
        click_num = extract_num(",".join(self['click_num']))
        crawl_time = datetime.datetime.now().strftime(SQL_DATETIME_FORMAT)
        params = (title,url,topics,zhihu_id,content,answer_num,comments_num,watch_user_num,click_num,crawl_time)

        return insert_sql, params

class ZhihuAnswerItem(scrapy.Item):
    question_id = scrapy.Field()
    url = scrapy.Field()
    author_id = scrapy.Field()
    zhihu_id = scrapy.Field()
    content = scrapy.Field()
    praise_num = scrapy.Field()
    comments_num = scrapy.Field()
    create_time = scrapy.Field()
    update_time = scrapy.Field()
    crawl_time = scrapy.Field()
    def get_sql(self):
        insert_sql = '''
                        insert into zhihu_answer(question_id,url,author_id,zhihu_id,
                        content,praise_num,comments_num,create_time,update_time,crawl_time)
                        VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) 
                        ON DUPLICATE KEY 
                        UPDATE praise_num=VALUES(praise_num),
                        comments_num=VALUES(comments_num),
                        update_time=VALUES(update_time),
                        crawl_time=VALUES(crawl_time)

                        '''
        question_id = self["question_id"]
        author_id = self['author_id']
        url = self["url"]
        zhihu_id = self['zhihu_id']
        content = self['content']
        praise_num = self['praise_num']
        comments_num =self['comments_num']
        create_time = datetime.datetime.fromtimestamp(self['create_time']).strftime(SQL_DATETIME_FORMAT)
        update_time = datetime.datetime.fromtimestamp(self['update_time']).strftime(SQL_DATETIME_FORMAT)
        crawl_time = self['crawl_time'].strftime(SQL_DATETIME_FORMAT)
        params = (question_id,url,author_id,zhihu_id,content,praise_num,comments_num,create_time,update_time,crawl_time)

        return insert_sql, params

pipeline中代码为:

class MysqlTwistedPipeline(object):
    def __init__(self,dbpool):
        self.dbpool = dbpool
    @classmethod
    def from_settings(cls,settings):
        dbparams = dict(
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DB"],
            user = settings["MYSQL_USER"],
            passwd = settings["MYSQL_PASSWD"],
            charset = "utf8",
            use_unicode = True,
            cursorclass = pymysql.cursors.DictCursor
        )
        dbpool = adbapi.ConnectionPool("pymysql",**dbparams)
        return cls(dbpool)

    def process_item(self, item, spider):
        query = self.dbpool.runInteraction(self.do_insert,item)
        query.addErrback(self.handle_error,item,spider)

    def handle_error(self, failure, item, spider):
        print(failure)

    def do_insert(self,cursor,item):
        insert_sql, params = item.get_sql()
        cursor.execute(insert_sql, tuple(params))

请问老师这是什么原因引起的呢?

写回答

1回答

bobby

2019-12-11

你留下qq 我加你看看

0
0

Scrapy打造搜索引擎 畅销4年的Python分布式爬虫课

带你彻底掌握Scrapy,用Django+Elasticsearch搭建搜索引擎

5796 学习 · 6290 问题

查看课程