AliExpress scraper that bills the page, not the product

import json
import os
import time

import requests

query = "wireless earbuds"
country = "us"


def scrape(url):
    for attempt in range(3):
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": url + (f"?_r={time.time_ns()}" if attempt else ""), "geo": country, "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if '"itemList":{"content":' in page:
            return page
        time.sleep(5)
    raise RuntimeError("No products came back, try again later")


def embedded(page, marker):
    start = page.index(marker) + len(marker)
    value, _ = json.JSONDecoder().raw_decode(page[start:])
    return value


search = "https://www.aliexpress.com/w/wholesale-" + "-".join(query.lower().split()) + ".html"
page = scrape(search)
products = []
for item in embedded(page, '"itemList":{"content":'):
    product_id = item.get("trace", {}).get("utLogMap", {}).get("x_object_id") or item["productId"]
    price = item.get("prices", {}).get("salePrice", {})
    products.append(
        {
            "id": product_id,
            "title": item["title"]["displayTitle"],
            "price": price.get("minPrice"),
            "currency": price.get("currencyCode"),
            "discount": price.get("discount"),
            "rating": item.get("evaluation", {}).get("starRating"),
            "sold": item.get("trade", {}).get("tradeDesc"),
            "ad": bool(item.get("p4p")),
            "url": f"https://www.aliexpress.com/item/{product_id}.html",
        }
    )

print(json.dumps({"query": query, "country": country, "products_on_page": len(products), "products": products}, indent=2, ensure_ascii=False))

import { writeFileSync } from 'node:fs';

const productId = '1005007823486360';

const scrape = async (url) => {
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const params = new URLSearchParams({ url: attempt ? `${url}&_r=${Date.now()}` : url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const html = (await response.json()).data?.html ?? '';
    const json = (html.match(/<pre[^>]*>([\s\S]*)<\/pre>/)?.[1] ?? html)
      .replace(/&quot;/g, '"')
      .replace(/&lt;/g, '<')
      .replace(/&gt;/g, '>')
      .replace(/&#39;/g, "'")
      .replace(/&amp;/g, '&');
    try {
      const data = JSON.parse(json).data;
      if (data?.evaViewList) return data;
    } catch {}
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error('No reviews came back, try again later');
};

const reviews = new Map();
let total = 0;
let pages = 1;

for (let page = 1; page <= pages; page += 1) {
  const data = await scrape(
    `https://feedback.aliexpress.com/pc/searchEvaluation.do?productId=${productId}&lang=en_US&country=US&page=${page}&pageSize=100&filter=all&sort=complex_default`,
  );
  total = data.totalNum;
  pages = data.totalPage;
  for (const review of data.evaViewList) {
    reviews.set(review.evaluationIdStr, {
      id: review.evaluationIdStr,
      rating: review.buyerEval / 20,
      date: review.evalDate,
      country: review.buyerCountry,
      variant: review.skuInfo?.trim() ?? '',
      text: review.buyerTranslationFeedback || review.buyerFeedback || '',
      photos: (review.images ?? []).join(' '),
    });
  }
}

const csv = (value) => `"${String(value).replace(/"/g, '""')}"`;
const rows = [...reviews.values()].map((r) => [r.id, r.rating, r.date, r.country, r.variant, r.text, r.photos].map(csv).join(','));
writeFileSync('reviews.csv', ['id,rating,date,country,variant,text,photos', ...rows].join('\n'));
console.log(`${reviews.size} of ${total} reviews from ${pages} requests saved to reviews.csv`);

const query = 'wireless earbuds';
const countries = ['us', 'gb', 'de', 'fr', 'es'];

const scrape = async (url, geo) => {
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const params = new URLSearchParams({ url: attempt ? `${url}?_r=${Date.now()}` : url, geo, format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const html = (await response.json()).data?.html ?? '';
    const at = html.indexOf('"itemList":{"content":');
    if (at < 0) {
      await new Promise((resolve) => setTimeout(resolve, 5000));
      continue;
    }
    const start = html.indexOf('[', at);
    let depth = 0;
    for (const match of html.slice(start).matchAll(/"(?:[^"\\]|\\.)*"|[[\]{}]/g)) {
      depth += '[{'.includes(match[0]) ? 1 : ']}'.includes(match[0]) ? -1 : 0;
      if (depth === 0) return JSON.parse(html.slice(start, start + match.index + 1));
    }
  }
  return [];
};

const search = `https://www.aliexpress.com/w/wholesale-${query.toLowerCase().split(' ').join('-')}.html`;
const seen = new Map();

for (const geo of countries) {
  const items = await scrape(search, geo);
  console.log(`${geo}: ${items.length} products in ${items[0]?.prices?.salePrice?.currencyCode ?? '-'}`);
  for (const item of items) {
    const id = item.trace?.utLogMap?.x_object_id ?? item.productId;
    const price = item.prices?.salePrice?.formattedPrice;
    if (!price) continue;
    seen.set(id, { title: seen.get(id)?.title ?? item.title.displayTitle, prices: { ...seen.get(id)?.prices, [geo]: price } });
  }
}

const shared = [...seen.entries()].filter(([, product]) => Object.keys(product.prices).length > 1);
console.log(`\n${shared.length} products found in more than one country:`);
for (const [id, product] of shared.slice(0, 5)) {
  console.log(`${id} ${product.title.slice(0, 50)}\n  ${Object.entries(product.prices).map(([geo, price]) => `${geo} ${price}`).join(' | ')}`);
}