どの都市のスポットもまとめて取得できるGoogle Mapsスクレイパー

const search = 'dentists in Miami';

const params = new URLSearchParams({
  url: `https://www.google.com/maps/search/${encodeURIComponent(search)}?hl=en&ucbcb=1`,
  render_js: 'true',
  wait: '9000',
  geo: 'us',
  format: 'json',
});

const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
  headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
});
const { data } = await response.json();
const text = (value) => value?.replaceAll('&', '&').replaceAll(''', "'").replaceAll('"', '"');

const places = data.html
  .split('role="article"')
  .slice(1)
  .map((card) => ({
    name: text(card.match(/class="hfpxzc" aria-label="([^"]+)"/)?.[1]),
    rating: card.match(/>([1-5]\.\d)</)?.[1],
    reviews: card.match(/>\(([\d,]+)\)</)?.[1],
    lat: card.match(/!3d(-?[\d.]+)/)?.[1],
    lng: card.match(/!4d(-?[\d.]+)/)?.[1],
  }));

console.table(places);

import html as entities
import os
import re
from urllib.parse import quote

import requests

search = "dentists in Miami"

response = requests.get(
    "https://parsing.webunlocker.gologin.com/v1/scrape",
    params={
        "url": f"https://www.google.com/maps/search/{quote(search)}?hl=en&ucbcb=1",
        "render_js": "true",
        "wait": "9000",
        "geo": "us",
        "format": "json",
    },
    headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
    timeout=90,
)
html = response.json()["data"]["html"]


def find(pattern, card):
    match = re.search(pattern, card)
    return entities.unescape(match.group(1)) if match else None


for card in html.split('role="article"')[1:]:
    print(
        find(r'class="hfpxzc" aria-label="([^"]+)"', card),
        find(r">([1-5]\.\d)<", card),
        find(r">\(([\d,]+)\)<", card),
        find(r"!3d(-?[\d.]+)", card),
        find(r"!4d(-?[\d.]+)", card),
    )

const query = 'dentists';
const center = { lat: 25.775, lng: -80.23 };
const step = 0.03;
const found = new Map();

async function tile(lat, lng) {
  const params = new URLSearchParams({
    url: `https://www.google.com/maps/search/${query}/@${lat},${lng},15z?hl=en&ucbcb=1`,
    render_js: 'true',
    wait: '9000',
    geo: 'us',
    format: 'json',
  });
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    if (response.ok) return (await response.json()).data.html;
    await new Promise((resolve) => setTimeout(resolve, 10000));
  }
  return '';
}

for (let row = -2; row <= 2; row += 1) {
  for (let col = -2; col <= 2; col += 1) {
    const html = await tile((center.lat + row * step).toFixed(4), (center.lng + col * step).toFixed(4));
    for (const card of html.split('role="article"').slice(1)) {
      const name = card.match(/class="hfpxzc" aria-label="([^"]+)"/)?.[1];
      if (name) found.set(name, { name, lat: card.match(/!3d(-?[\d.]+)/)?.[1], lng: card.match(/!4d(-?[\d.]+)/)?.[1] });
    }
    console.log(`tile ${row},${col}: ${found.size} places so far`);
  }
}

console.table([...found.values()]);