Tripadvisor scraper that bills the page, not the review

import json
import os
import re
import time
from html import unescape

import requests

address = "https://www.tripadvisor.com/Hotel_Review-g60745-d8316858-Reviews-The_Godfrey_Hotel_Boston-Boston_Massachusetts.html"


def scrape(url):
    for attempt in range(3):
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            params={
                "url": url if attempt == 0 else f"{url}?_r={int(time.time())}",
                "geo": "us",
                "format": "json",
            },
            timeout=120,
        )
        html = response.json().get("data", {}).get("html", "")
        if "ShowUserReviews-" in html:
            return html
    return html


def text(markup):
    return unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", markup.replace("<!-- -->", "")))).strip()


def first(pattern, source):
    match = re.search(pattern, source)
    return text(match.group(1)) if match else None


html = scrape(address)

place = {}
for block in re.findall(r'<script[^>]*application/ld\+json[^>]*>(.*?)</script>', html, re.S):
    data = json.loads(block)
    if "aggregateRating" in data:
        place = data
        break

cards = re.split(r'data-test-target="HR_CC_CARD"|data-automation="reviewCard"', html)[1:]
reviews = []
for card in cards:
    title = first(r'<a[^>]+ShowUserReviews[^>]+>(.*?)</a>', card)
    if not title:
        continue
    body = card.split("</h3>", 1)[1]
    body = body[body.find('data-test-target="review-body"'):] if 'data-test-target="review-body"' in body else body
    reviews.append({
        "author": first(r'href="/Profile/[^"]+"[^>]*>(?:<span[^>]*>)?([^<]+)<', card),
        "rating": first(r"<title[^>]*>([\d.]+) of 5 bubbles</title>", card),
        "title": title,
        "text": text(re.split(r"<button|Date of stay:|<div[^>]*>Written ", body)[0]),
        "date_of_stay": first(r"Date of stay:(?:<[^>]+>)+([^<]+)<", card),
        "owner_reply": "Response from " in card,
    })

next_page = re.search(r'aria-label="Next page" href="([^"]+)"', html)
print(json.dumps({
    "name": place.get("name"),
    "rating": place.get("aggregateRating", {}).get("ratingValue"),
    "reviews_total": place.get("aggregateRating", {}).get("reviewCount"),
    "ranking": first(r"(#[\d,]+ of [\d,]+ [^<\"]+?)(?= and rated|\"|<)", html),
    "reviews": reviews,
    "next_page": f"https://www.tripadvisor.com{unescape(next_page.group(1))}" if next_page else None,
}, indent=2, ensure_ascii=False))

import { writeFileSync } from 'node:fs';

let address = 'https://www.tripadvisor.com/Restaurant_Review-g60763-d33984348-Reviews-Olio_e_Piu-New_York_City_New_York.html';

const scrape = async (url) => {
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const params = new URLSearchParams({ url: attempt ? `${url}?_r=${Date.now()}` : url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const html = (await response.json()).data?.html ?? '';
    if (html.includes('ShowUserReviews-')) return html;
  }
  return '';
};

const text = (markup) =>
  markup
    .replace(/<[^>]+>/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&#39;|&#x27;/g, "'")
    .replace(/&quot;/g, '"')
    .replace(/\s+/g, ' ')
    .trim();

const reviews = new Map();
let pages = 0;

while (address) {
  const html = await scrape(address);
  pages += 1;
  for (const card of html.split(/data-test-target="HR_CC_CARD"|data-automation="reviewCard"/).slice(1)) {
    const id = card.match(/ShowUserReviews-g\d+-d\d+-r(\d+)/)?.[1];
    if (!id) continue;
    const at = card.indexOf('data-test-target="review-body"');
    const body = at >= 0 ? card.slice(at) : card.slice(card.indexOf('</h3>'));
    reviews.set(id, {
      id,
      rating: card.match(/<title[^>]*>([\d.]+) of 5 bubbles<\/title>/)?.[1],
      title: text(card.match(/<a[^>]+ShowUserReviews[^>]+>([\s\S]*?)<\/a>/)?.[1] ?? ''),
      written: card.match(/Written (?:<!-- -->)?([^<]+)</)?.[1] ?? '',
      text: text(body.slice(body.indexOf('>') + 1).split(/<button|Date of stay:|<div[^>]*>Written /)[0]),
    });
  }
  const next = html.match(/aria-label="Next page" href="([^"]+)"/)?.[1];
  address = next ? `https://www.tripadvisor.com${next.replace(/&amp;/g, '&')}` : null;
}

const csv = (value) => `"${String(value).replace(/"/g, '""')}"`;
const rows = [...reviews.values()].map((r) => [r.id, r.rating, r.written, r.title, r.text].map(csv).join(','));
writeFileSync('reviews.csv', ['id,rating,written,title,text', ...rows].join('\n'));
console.log(`${reviews.size} reviews from ${pages} pages saved to reviews.csv`);

const address = 'https://www.tripadvisor.com/Hotels-g60745-Boston_Massachusetts-Hotels.html';

const params = new URLSearchParams({ url: address, geo: 'us', format: 'json' });
const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
  headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
});
const html = (await response.json()).data?.html ?? '';

const hotels = [];
for (const [, json] of html.matchAll(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/g)) {
  const data = JSON.parse(json);
  if (data['@type'] !== 'ItemList') continue;
  for (const { item } of data.itemListElement) {
    hotels.push({
      name: item.name,
      rating: item.aggregateRating?.ratingValue,
      reviews: item.aggregateRating?.reviewCount,
      phone: item.telephone,
      address: item.address?.streetAddress,
      page: item.url.replace(/\?.*$/, '').replace('/HotelHighlight-', '/Hotel_Review-'),
    });
  }
}

console.table(hotels.map(({ name, rating, reviews, phone }) => ({ name, rating, reviews, phone })));
console.log(`${hotels.length} hotels, each with its page for the reviews`);