レビューごと​ではなく​店舗単位で​課金する​Googleレビュー​スクレイパー

import json
import os
import re
import time
from urllib.parse import quote_plus

import requests

place = "Franklin Barbecue Austin"
address = f"https://www.google.com/maps/search/{quote_plus(place)}?hl=en&ucbcb=1"

html = ""
for attempt in range(3):
    response = requests.get(
        "https://parsing.webunlocker.gologin.com/v1/scrape",
        headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
        params={
            "url": address if attempt == 0 else f"{address}&_r={int(time.time())}",
            "render_js": "true",
            "wait": "9000",
            "geo": "us",
            "format": "json",
        },
        timeout=120,
    )
    html = response.json().get("data", {}).get("html", "")
    if 'data-review-id="' in html:
        break

def first(pattern):
    match = re.search(pattern, html)
    return match.group(1) if match else None

blocks = {}
for match in re.finditer(r'data-review-id="([^"]+)"', html):
    blocks.setdefault(match.group(1), match.start())
starts = sorted(blocks.values()) + [len(html)]

reviews = []
for start, end in zip(starts, starts[1:]):
    block = html[start:end]
    author = re.search(r'class="d4r55[^"]*"[^>]*>([^<]+)<', block)
    text = re.search(r'class="wiI7pd"[^>]*>([^<]*)<', block)
    if author and text:
        stars = re.search(r'aria-label="(\d) stars?"', block)
        date = re.search(r'class="rsqaWe"[^>]*>([^<]+)<', block)
        reviews.append({
            "author": author.group(1),
            "stars": int(stars.group(1)) if stars else None,
            "date": date.group(1) if date else None,
            "text": text.group(1),
        })

print(json.dumps({
    "name": first(r"<h1[^>]*>([^<]+)"),
    "rating": first(r'class="F7nice[^"]*"[^>]*>\s*<span><span aria-hidden="true">([\d.]+)<'),
    "ratings": first(r'aria-label="([\d,]+) reviews?">\('),
    "stars": {level: first(rf'aria-label="{level} stars?, ([\d,]+) reviews?"') for level in "54321"},
    "topics": dict(re.findall(r'aria-label="([^",]+), mentioned in ([\d,]+) reviews?"', html)),
    "reviews": reviews,
}, indent=2, ensure_ascii=False))

const places = [
  'Franklin Barbecue Austin',
  'Katz Delicatessen New York',
  'Relax and Smile Dental Care Miami',
];

const number = (text) => (text ? Number(text.replace(/,/g, '')) : null);

async function pageOf(place, attempt) {
  const address = `https://www.google.com/maps/search/${encodeURIComponent(place).replace(/%20/g, '+')}?hl=en&ucbcb=1`;
  const params = new URLSearchParams({
    url: attempt === 1 ? address : `${address}&_r=${Date.now()}`,
    render_js: 'true',
    wait: '9000',
    geo: 'us',
    format: 'json',
  });
  const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
    headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
  });
  return (await response.json()).data?.html ?? '';
}

async function reviewsOf(place) {
  let html = '';
  for (let attempt = 1; attempt <= 3 && !html.includes('data-review-id='); attempt += 1) {
    html = await pageOf(place, attempt);
  }
  const star = (level) => number(html.match(new RegExp(`aria-label="${level} stars?, ([\\d,]+) reviews?"`))?.[1]);
  return {
    place: html.match(/<h1[^>]*>([^<]+)/)?.[1] ?? place,
    rating: number(html.match(/class="F7nice[^"]*"[^>]*>\s*<span><span aria-hidden="true">([\d.]+)</)?.[1]),
    ratings: number(html.match(/aria-label="([\d,]+) reviews?">\(/)?.[1]),
    fiveStar: star(5),
    oneStar: star(1),
    topReview: html.match(/class="wiI7pd"[^>]*>([^<]{0,80})/)?.[1],
  };
}

const rows = [];
for (const place of places) rows.push(await reviewsOf(place));
console.table(rows);

import { readFile, writeFile } from 'node:fs/promises';

const places = ['Franklin Barbecue Austin', 'Katz Delicatessen New York', 'Relax and Smile Dental Care Miami'];
const FILE = 'reviews.json';

const number = (text) => (text ? Number(text.replace(/,/g, '')) : null);

async function pageOf(place, attempt) {
  const address = `https://www.google.com/maps/search/${encodeURIComponent(place).replace(/%20/g, '+')}?hl=en&ucbcb=1`;
  const params = new URLSearchParams({
    url: attempt === 1 ? address : `${address}&_r=${Date.now()}`,
    render_js: 'true',
    wait: '9000',
    geo: 'us',
    format: 'json',
  });
  const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
    headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
  });
  return (await response.json()).data?.html ?? '';
}

async function countOf(place) {
  let html = '';
  for (let attempt = 1; attempt <= 3 && !html.includes('data-review-id='); attempt += 1) {
    html = await pageOf(place, attempt);
  }
  return {
    rating: number(html.match(/class="F7nice[^"]*"[^>]*>\s*<span><span aria-hidden="true">([\d.]+)</)?.[1]),
    ratings: number(html.match(/aria-label="([\d,]+) reviews?">\(/)?.[1]),
  };
}

const before = JSON.parse(await readFile(FILE, 'utf8').catch(() => '{}'));
const today = { ...before };

for (const place of places) {
  const now = await countOf(place);
  const was = before[place]?.ratings ?? null;
  if (now.ratings === null) console.log(`${place}: no page today, ask again`);
  else if (was === null) console.log(`${place}: first check, ${now.ratings} ratings, ${now.rating} stars`);
  else if (now.ratings > was) console.log(`${place}: ${now.ratings - was} new, now ${now.ratings} ratings, ${now.rating} stars`);
  else console.log(`${place}: nothing new`);
  if (now.ratings !== null) today[place] = now;
}

await writeFile(FILE, JSON.stringify(today, null, 2));