Scrapy找不到项目

Question

我目前仍在学习Scrapy，并尝试使用管道和ItemLoader。

但是，我目前遇到的问题是，爬行器显示Item.py不存在。我到底做错了什么，为什么我没有从爬行器获得任何数据到我的管道中？

在不导入项目的情况下运行爬行器可以很好地工作。管道也在settings.py中激活。

我的错误日志如下：

Traceback (most recent call last):
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\Scripts\scrapy.exe\__main__.py", line 7, in <module> 
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\cmdline.py", line 144, in execute
    cmd.crawler_process = CrawlerProcess(settings)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\crawler.py", line 280, in __init__
    super().__init__(settings)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\crawler.py", line 152, in __init__
    self.spider_loader = self._get_spider_loader(settings)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\crawler.py", line 146, in _get_spider_loader
    return loader_cls.from_settings(settings.frozencopy())
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\spiderloader.py", line 67, in from_settings
    return cls(settings)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\spiderloader.py", line 24, in __init__
    self._load_all_spiders()
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\spiderloader.py", line 51, in _load_all_spiders
    for module in walk_modules(name):
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\site-packages\scrapy\utils\misc.py", line 88, in 
walk_modules
    submod = import_module(fullpath)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\importlib\__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1006, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 688, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 883, in exec_module
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
  File "C:\Users\Syrix\WebCrawler\watches\watches\spiders\watchbot.py", line 5, in <module>
    from watches.watches.items import WatchesItem
ModuleNotFoundError: No module named 'watches.watches'

我的蜘蛛看起来是这样的：

import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.loader import ItemLoader
from watches.watches.items import WatchesItem
from scrapy.exceptions import DropItem


class WatchbotSpider(scrapy.Spider):
    name = 'watchbot'
    start_urls = ['https://www.watch.de/english/rolex.html']


    def parse(self, response, **kwargs):
        for link in response.css('div.product-item-link a::attr(href)'):
            url = link.get()
            yield scrapy.Request(url, callback=self.parse_categories)


    def parse_categories(self, response):
        for product in response.xpath('//*[@id="main"]/div[2]/div[1]'):
            l = ItemLoader(item=WatchesItem(), selector=product)
            l.add_xpath('name', '//span[@itemprop="sku"]/text()')
            l.add_xpath('reference', '//span[@itemprop="sku"]/text()')
            l.add_xpath('year', '//div[@class="product-option baujahr"]/div[@class="product-option-value"]/text()')

            yield l.load_item()

items.py：

import scrapy
from scrapy.loader import ItemLoader
from itemloaders.processors import TakeFirst, MapCompose
from w3lib.html import remove_tags


class WatchesItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field(input_processor = MapCompose(remove_tags, output_processor = TakeFirst()))
    reference = scrapy.Field(input_processor = MapCompose(remove_tags, output_processor = TakeFirst()))
    year = scrapy.Field(input_processor = MapCompose(remove_tags, output_processor = TakeFirst()))

最后但并非最不重要的是我的流水线：

import mysql
import mysql.connector
from watches.watches.spiders import watchbot

class WatchesPipeline(object):

    def __init__(self):
        self.conn = mysql.connector.connect(
            host = '',
            user = '',
            passwd = '',
            database = ''
        )
        self.curr = self.conn.cursor()


    def process_item(self, item, spider):
        self.store_db(item)
        print("Pipleline = " + item['name'] + " " + item['reference'] + " " + item['year'])
        return item

    def store_db(self, item):
        self.curr.execute("""insert into test.watch values (%s, %s, %s)""", (
            item['name'][0],
            item['reference'][0],
            item['year'][0],
        ))
        self.conn.commit()

编辑：

PS E:\semester\webcrawler_watches\watches\Crawler> scrapy crawl watchbot
Traceback (most recent call last):
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "E:\semester\webcrawler_watches\venv\Scripts\scrapy.exe\__main__.py", line 7, in <module>
  File "E:\semester\webcrawler_watches\venv\lib\site-packages\scrapy\cmdline.py", line 114, in execute
    settings = get_project_settings()
  File "E:\semester\webcrawler_watches\venv\lib\site-packages\scrapy\utils\project.py", line 68, in get_project_settings
    settings.setmodule(settings_module_path, priority='project')
  File "E:\semester\6. semester\webcrawler_watches\venv\lib\site-packages\scrapy\settings\__init__.py", line 287, in setmodule
    module = import_module(module)
  File "C:\Users\Syrix\AppData\Local\Programs\Python\Python310\lib\importlib\__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 992, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1004, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'watches'
PS E:\semester\webcrawler_watches\watches\Crawler>

Answer 1

这是我的工作。请跟着这个走。

import mysql
import mysql.connector
# from watches.watches.spiders import watchbot

class WatchesPipeline(object):

    def __init__(self):
        self.conn = mysql.connector.connect(
            host = 'localhost', 
            user = 'root',
            passwd = '', # your password
            database = '', # your databse
           
        )
        self.curr = self.conn.cursor()


    # def create_table(self):
    #     self.curr.execute("""DROP TABLE IF EXISTS scrapy_tb """)
    #     self.curr.execute("""create table scrapy_tb (name text, reference text, year text)""")


    def process_item(self, item, spider):
        self.store_db(item)
        # print("Pipleline = " + item['name'] + " " + item['reference'] + " " + item['year'])
        return item

    def store_db(self, item):
        self.curr.execute("""insert into scrapy_tb values (%s, %s, %s)""", 
            (
            item['name'][0],
            item['reference'][0],
            item['year'][0]
        ))
        self.conn.commit()
        return item
        self.conn.close()