Ggtcf

I've written a script using scrapy to get the name, phone number and email from a website. The content I'm after are available in two diferent links, as in name and phone in one link and the email is in another link. I've used here yellowpages.com as an example and tried to implement the logic in such a way so that I can parse the email even when I'm in it's landing page. It's a requirement that I can't use meta. However, I used requests and BeautifulSoup in combination with scrapy to accomplish the job complying the above condition but it is real slow.

Working one (along with requests and BeautifulSoup):

import scrapy
import requests
from bs4 import BeautifulSoup
from scrapy.crawler import CrawlerProcess

def get_email(target_link):
 res = requests.get(target_link)
 soup = BeautifulSoup(res.text,"lxml")
 email = soup.select_one("a.email-business[href^='mailto:']")
 if email:
 return email.get("href")
 else:
 return None

class YellowpagesSpider(scrapy.Spider):
 name = "yellowpages"
 start_urls = ["https://www.yellowpages.com/search?search_terms=Coffee+Shops&geo_location_terms=San+Francisco%2C+CA"]

 def parse(self,response):
 for items in response.css("div.v-card .info"):
 name = items.css("a.business-name > span::text").get()
 phone = items.css("div.phones::text").get()
 email = get_email(response.urljoin(items.css("a.business-name::attr(href)").get()))
 yield "Name":name,"Phone":phone,"Email":email

if __name__ == "__main__":
 c = CrawlerProcess(
 'USER_AGENT': 'Mozilla/5.0',
 )
 c.crawl(YellowpagesSpider)
 c.start()

I'm trying to mimic the above concept without requests and BeautifulSoup but can't make it work.

import scrapy
from scrapy.crawler import CrawlerProcess

class YellowpagesSpider(scrapy.Spider):
 name = "yellowpages"
 start_urls = ["https://www.yellowpages.com/search?search_terms=Coffee+Shops&geo_location_terms=San+Francisco%2C+CA"]

 def parse(self,response):
 for items in response.css("div.v-card .info"):
 name = items.css("a.business-name > span::text").get()
 phone = items.css("div.phones::text").get()
 email_link = response.urljoin(items.css("a.business-name::attr(href)").get())

 #CANT APPLY THE LOGIC IN THE FOLLOWING LINE

 email = self.get_email(email_link)
 yield "Name":name,"Phone":phone,"Email":email

 def get_email(self,link):
 email = response.css("a.email-business[href^='mailto:']::attr(href)").get()
 return email

if __name__ == "__main__":
 c = CrawlerProcess(
 'USER_AGENT': 'Mozilla/5.0',
 )
 c.crawl(YellowpagesSpider)
 c.start()

How can I make my second script work mimicking the first script?

I'd use response.meta, but if is required to avoid it, ok, let's try in another way: check lib https://pypi.org/project/scrapy-inline-requests/

from inline_requests import inline_requests


class YellowpagesSpider(scrapy.Spider):
 name = "yellowpages"
 start_urls = ["https://www.yellowpages.com/search?search_terms=Coffee+Shops&geo_location_terms=San+Francisco%2C+CA"]

 @inline_requests
 def parse(self, response):
 for items in response.css("div.v-card .info"):
 name = items.css("a.business-name > span::text").get()
 phone = items.css("div.phones::text").get()

 email_url = items.css("a.business-name::attr(href)").get()
 email_resp = yield scrapy.Request(response.urljoin(email_url), meta='handle_httpstatus_all': True)
 email = email_resp.css("a.email-business[href^='mailto:']::attr(href)").get() if email_resp.status == 200 else None
 yield "Name": name, "Phone": phone, "Email": email