Google Play scraper with ratings, installs and reviews in one request

import json
import os
import re
import time
from datetime import datetime, timezone

import requests

app_id = "com.duolingo"
country = "us"


def scrape(url):
    for attempt in range(3):
        address = url if attempt == 0 else f"{url}&fresh={int(time.time())}"
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": address, "geo": country, "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if "key: 'ds:5'" in page:
            return page
        time.sleep(3)
    raise RuntimeError("The page came back without the app data, try again later")


def blocks(page):
    for key, data in re.findall(r"AF_initDataCallback\(\{key: '(ds:\d+)'.*?data:(.*?), sideChannel: \{\}\}\);</script>", page, re.S):
        yield key, json.loads(data)


def at(node, *path):
    for step in path:
        if not isinstance(node, list) or step >= len(node):
            return None
        node = node[step]
    return node


def day(stamp):
    return datetime.fromtimestamp(stamp[0], timezone.utc).date().isoformat() if stamp else None


page = scrape(f"https://play.google.com/store/apps/details?id={app_id}&hl=en&gl={country.upper()}")
data = dict(blocks(page))
d = at(data["ds:5"], 1, 2)
reviews = next(
    (value[0] for value in data.values() if isinstance(at(value, 0, 0, 0), str) and re.match(r"^[0-9a-f]{8}-", at(value, 0, 0, 0))),
    [],
)

app = {
    "id": app_id,
    "title": at(d, 0, 0),
    "developer": at(d, 68, 0),
    "genre": at(d, 79, 0, 0, 0),
    "rating": at(d, 51, 0, 1),
    "ratings": at(d, 51, 2, 1),
    "installs": at(d, 13, 0),
    "installs_exact": at(d, 13, 2),
    "price": (at(d, 57, 0, 0, 0, 0, 1, 0, 0) or 0) / 1e6,
    "currency": at(d, 57, 0, 0, 0, 0, 1, 0, 1),
    "version": at(d, 140, 0, 0, 0) or "Varies with device",
    "updated": at(d, 145, 0, 0),
    "reviews": [
        {
            "author": at(review, 1, 0),
            "stars": at(review, 2),
            "text": at(review, 4),
            "date": day(at(review, 5)),
            "helpful": at(review, 6),
            "app_version": at(review, 10),
            "reply": at(review, 7, 1),
        }
        for review in reviews
    ],
}

print(json.dumps(app, indent=2, ensure_ascii=False))
print(f"{app['title']}: {app['rating']:.2f} from {app['ratings']:,} ratings, {app['installs_exact']:,} installs, {len(app['reviews'])} reviews from one page")

import { writeFileSync } from 'node:fs';

const apps = ['com.duolingo', 'com.babbel.mobile.android.en', 'com.memrise.android.memrisecompanion', 'com.busuu.android.enc', 'com.hellotalk'];
const country = 'us';

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}&fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo: country, format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes("key: 'ds:5'")) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return '';
}

const at = (node, ...path) => path.reduce((value, step) => (Array.isArray(value) ? value[step] : undefined), node);
const appData = (page) => {
  const block = page.match(/AF_initDataCallback\(\{key: 'ds:5'.*?data:(.*?), sideChannel: \{\}\}\);<\/script>/s)?.[1];
  return block ? at(JSON.parse(block), 1, 2) : null;
};

const rows = [];
for (const id of apps) {
  const d = appData(await scrape(`https://play.google.com/store/apps/details?id=${id}&hl=en&gl=${country.toUpperCase()}`));
  rows.push({
    id,
    title: at(d, 0, 0),
    rating: at(d, 51, 0, 1)?.toFixed(3),
    ratings: at(d, 51, 2, 1),
    installs: at(d, 13, 2),
    version: at(d, 140, 0, 0, 0) ?? 'Varies with device',
    updated: at(d, 145, 0, 0),
    price: (at(d, 57, 0, 0, 0, 0, 1, 0, 0) ?? 0) / 1e6,
  });
}

writeFileSync('apps.csv', ['id,title,rating,ratings,installs,version,updated,price', ...rows.map((row) => Object.values(row).map((value) => JSON.stringify(value ?? '')).join(','))].join('\n'));
console.table(rows);
console.log(`${rows.length} apps in ${rows.length} pages: $${(rows.length * 0.00098).toFixed(4)} on the Growth plan`);

const keyword = 'meditation';
const app = 'com.calm.android';
const countries = ['us', 'gb', 'de', 'ca', 'au'];

async function scrape(url, geo) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}&fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo, format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes("key: 'ds:4'")) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return '';
}

const results = (page) => {
  const block = page.match(/AF_initDataCallback\(\{key: 'ds:4'.*?data:(.*?), sideChannel: \{\}\}\);<\/script>/s)?.[1];
  const sections = block ? JSON.parse(block)?.[0]?.[1] : null;
  const items = sections?.find((section) => section?.[22]?.[0])?.[22]?.[0];
  return (items ?? []).map(([item]) => ({ id: item[0][0], title: item[3], rating: item[4]?.[1] }));
};

for (const country of countries) {
  const found = results(await scrape(`https://play.google.com/store/search?q=${encodeURIComponent(keyword)}&c=apps&hl=en&gl=${country.toUpperCase()}`, country));
  const place = found.findIndex((item) => item.id === app) + 1;
  console.log(`${country.toUpperCase()}: ${app} is #${place || '-'} of ${found.length} for "${keyword}", top app: ${found[0]?.title} (${found[0]?.rating?.toFixed(1)})`);
}
console.log(`${countries.length} countries in ${countries.length} pages: $${(countries.length * 0.00098).toFixed(4)} on the Growth plan`);