How Can I Get Some Attributes From A Website For Scraping

I have an issue while trying to scrape a website. I have tried a couple of methods to get the restaurant_name, cuisine, address and star ratings but I keep getting the error 'NoneT

Solution 1:

Here's how I was able to do it, not sure if it is the most efficient but it definitely works:

import requests
from bs4 import BeautifulSoup
import re

city = input("Enter your city: ")
url = "https://www.zomato.com/" + city + "/top-restaurants"
header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"}
response = requests.get(url, headers=header)

html = response.text
soup = BeautifulSoup(html, "html.parser")
print(soup.title.text)

top_rest = soup.find("div", class_="bke1zw-0 cMipmx")

restaurants = []

restaurant_divs = top_rest.select("div > section > div")

for rdiv in restaurant_divs:
    name = rdiv.find("a", recursive=False).text.strip()
    rating_div, address_div, cuisine_div = rdiv.find_all("div", recursive=False)
    ratings = re.findall(r"([\d\.]+)\(([\d,]+)\)", rating_div.text)
    black_rating = (float(ratings[0][0]), int(ratings[0][1].replace(',', ''))) if ratings else (None, None)
    red_rating = (float(ratings[1][0]), int(ratings[1][1].replace(',', ''))) iflen(ratings) > 1else (None, None)
    restaurants.append({"name": name,
        "ratings": {
            "black": {
                "score": black_rating[0],
                "votes": black_rating[1]
                },
            "red": {
                "score": red_rating[0],
                "votes": red_rating[1]
                }
            },
        "location": address_div.text,
        "cuisine": cuisine_div.text
        })

Note that some unicode characters aren't displayed correctly, but that can be fixed later.

I also took some liberty and reformatted the response, please take a look:

Top Restaurants in Kanpur | Zomato
[
  {
    "name": "Grill Inn",
    "ratings": {
      "black": {
        "score": 3.4,
        "votes": 77
      },
      "red": {
        "score": 4.1,
        "votes": 1668
      }
    },
    "location": "Shyam Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0Italian"
  },
  {
    "name": "Shri Bhojnalaya Restaurant & Sweets",
    "ratings": {
      "black": {
        "score": 4.2,
        "votes": 492
      },
      "red": {
        "score": 3.8,
        "votes": 8151
      }
    },
    "location": "Vijay Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0North Indian,\u00a0Chinese,\u00a0Fast Fo
od,\u00a0Desserts"
  },
  {
    "name": "Barbeque Nation",
    "ratings": {
      "black": {
        "score": 4.9,
        "votes": 716
      },
      "red": {
        "score": 4.1,
        "votes": 275
      }
    },
    "location": "Mall Road",
    "cuisine": "Casual Dining\u00a0-\u00a0North Indian,\u00a0Mughlai,\u00a0Leban
ese,\u00a0Arabian,\u00a0Mediterranean"
  },
  {
    "name": "Kukkad at Nukkad",
    "ratings": {
      "black": {
        "score": 3.3,
        "votes": 272
      },
      "red": {
        "score": 4.2,
        "votes": 500
      }
    },
    "location": "Swaroop Nagar",
    "cuisine": "Casual Dining\u00a0-\u00a0Mughlai,\u00a0North Indian"
  },
  {
    "name": "Tadka The Food Hub",
    "ratings": {
      "black": {
        "score": 3.8,
        "votes": 181
      },
      "red": {
        "score": 3.9,
        "votes": 6594
      }
    },
    "location": "Kidwai Nagar Market",
    "cuisine": "Quick Bites\u00a0-\u00a0Chinese,\u00a0North Indian,\u00a0South I
ndian"
  },
  {
    "name": "Smile Pizza",
    "ratings": {
      "black": {
        "score": 3.8,
        "votes": 94
      },
      "red": {
        "score": 4.0,
        "votes": 3142
      }
    },
    "location": "Kidwai Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0Pizza,\u00a0Fast Food"
  },
  {
    "name": "Arabian Broost Chicken",
    "ratings": {
      "black": {
        "score": 4.3,
        "votes": 420
      },
      "red": {
        "score": 4.2,
        "votes": 7051
      }
    },
    "location": "Chamanganj",
    "cuisine": "Quick Bites\u00a0-\u00a0Arabian"
  },
  {
    "name": "Chachi Vaishno Dhaba",
    "ratings": {
      "black": {
        "score": 4.2,
        "votes": 346
      },
      "red": {
        "score": 3.9,
        "votes": 6354
      }
    },
    "location": "Nandlal Chawraha",
    "cuisine": "Dhaba\u00a0-\u00a0North Indian"
  },
  {
    "name": "Barra House",
    "ratings": {
      "black": {
        "score": 4.2,
        "votes": 274
      },
      "red": {
        "score": 3.8,
        "votes": 6233
      }
    },
    "location": "Kanpur Cantt",
    "cuisine": "Quick Bites\u00a0-\u00a0North Indian,\u00a0Mughlai"
  },
  {
    "name": "Pashtun's",
    "ratings": {
      "black": {
        "score": 3.7,
        "votes": 28
      },
      "red": {
        "score": 3.9,
        "votes": 254
      }
    },
    "location": "Swaroop Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0Kebab,\u00a0North Indian"
  },
  {
    "name": "Agra Vala Sweets",
    "ratings": {
      "black": {
        "score": 4.0,
        "votes": 174
      },
      "red": {
        "score": 4.4,
        "votes": 5495
      }
    },
    "location": "Ashok Nagar",
    "cuisine": "Quick Bites,\u00a0Sweet Shop\u00a0-\u00a0Street Food,\u00a0Mitha
i"
  },
  {
    "name": "Al-Baik.Com",
    "ratings": {
      "black": {
        "score": 3.9,
        "votes": 77
      },
      "red": {
        "score": null,
        "votes": null
      }
    },
    "location": "Shyam Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0Fast Food"
  },
  {
    "name": "The Imperial Cord",
    "ratings": {
      "black": {
        "score": 3.8,
        "votes": 2519
      },
      "red": {
        "score": null,
        "votes": null
      }
    },
    "location": "Kakadeo",
    "cuisine": "Quick Bites\u00a0-\u00a0Fast Food,\u00a0North Indian"
  },
  {
    "name": "Google Fast Food",
    "ratings": {
      "black": {
        "score": 4.1,
        "votes": 144
      },
      "red": {
        "score": 4.2,
        "votes": 2006
      }
    },
    "location": "Nandlal Chawraha",
    "cuisine": "Quick Bites\u00a0-\u00a0Fast Food,\u00a0Chinese,\u00a0North Indi
an"
  },
  {
    "name": "Baba Foods",
    "ratings": {
      "black": {
        "score": 4.5,
        "votes": 1229
      },
      "red": {
        "score": 4.2,
        "votes": 2668
      }
    },
    "location": "Swaroop Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0North Indian,\u00a0Biryani,\u00a0Beverag
es,\u00a0Desserts"
  },
  {
    "name": "R S Bhojnalaya",
    "ratings": {
      "black": {
        "score": 4.2,
        "votes": 530
      },
      "red": {
        "score": null,
        "votes": null
      }
    },
    "location": "Kakadeo",
    "cuisine": "Bhojanalya\u00a0-\u00a0North Indian"
  },
  {
    "name": "Kerela Cafe",
    "ratings": {
      "black": {
        "score": 4.1,
        "votes": 299
      },
      "red": {
        "score": 3.9,
        "votes": 4191
      }
    },
    "location": "IIT Kanpur",
    "cuisine": "Quick Bites\u00a0-\u00a0South Indian"
  },
  {
    "name": "Mama Hotel",
    "ratings": {
      "black": {
        "score": 4.1,
        "votes": 364
      },
      "red": {
        "score": null,
        "votes": null
      }
    },
    "location": "Swaroop Nagar",
    "cuisine": "Quick Bites\u00a0-\u00a0North Indian,\u00a0Beverages"
  },
  {
    "name": "Gyan Vaishnav",
    "ratings": {
      "black": {
        "score": 4.6,
        "votes": 927
      },
      "red": {
        "score": null,
        "votes": null
      }
    },
    "location": "Ashok Nagar",
    "cuisine": "Casual Dining\u00a0-\u00a0North Indian"
  },
  {
    "name": "New Pizza Yum",
    "ratings": {
      "black": {
        "score": 3.8,
        "votes": 201
      },
      "red": {
        "score": 4.1,
        "votes": 2624
      }
    },
    "location": "Kakadeo",
    "cuisine": "Quick Bites\u00a0-\u00a0Pizza"
  },
  {
    "name": "Offline Cafe",
    "ratings": {
      "black": {
        "score": 4.1,
        "votes": 624
      },
      "red": {
        "score": 3.7,
        "votes": 1995
      }
    },
    "location": "Tilak Nagar",
    "cuisine": "Caf\u00e9\u00a0-\u00a0Cafe,\u00a0North Indian,\u00a0Fast Food"
  },
  {
    "name": "The Chocolate Room",
    "ratings": {
      "black": {
        "score": 4.0,
        "votes": 463
      },
      "red": {
        "score": 4.0,
        "votes": 5868
      }
    },
    "location": "Swaroop Nagar",
    "cuisine": "Caf\u00e9,\u00a0Dessert Parlor\u00a0-\u00a0Cafe,\u00a0Desserts"
  },
  {
    "name": "Mocha",
    "ratings": {
      "black": {
        "score": 4.7,
        "votes": 1146
      },
      "red": {
        "score": 4.0,
        "votes": 1138
      }
    },
    "location": "Mall Road",
    "cuisine": "Caf\u00e9,\u00a0Casual Dining\u00a0-\u00a0Cafe"
  }
]