Web scraping dinâmico para páginas que carregam os dados depois

import { chromium } from "playwright-core";

const token = process.env.GOLOGIN_TOKEN;
const profileId = process.env.GOLOGIN_PROFILE_ID;
const endpoint = `wss://cloudbrowser.gologin.com/connect?token=${token}&profile=${profileId}`;

const browser = await chromium.connectOverCDP(endpoint);
const page = await browser.contexts()[0].newPage();

await page.goto("http://quotes.toscrape.com/scroll");
await page.waitForSelector(".quote");

for (let i = 0; i < 30; i++) {
  const before = await page.locator(".quote").count();
  await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
  try {
    await page.waitForFunction(
      (n) => document.querySelectorAll(".quote").length > n,
      before,
      { timeout: 3000 }
    );
  } catch {
    break;
  }
}

const quotes = await page.$$eval(".quote", (els) =>
  els.map((el) => ({
    text: el.querySelector(".text")?.textContent,
    author: el.querySelector(".author")?.textContent,
  }))
);
console.log(quotes.length, quotes[0]);

await page.close();
await browser.close();

import puppeteer from "puppeteer-core";

const token = process.env.GOLOGIN_TOKEN;
const profileId = process.env.GOLOGIN_PROFILE_ID;
const endpoint = `wss://cloudbrowser.gologin.com/connect?token=${token}&profile=${profileId}`;

const browser = await puppeteer.connect({ browserWSEndpoint: endpoint });
const page = await browser.newPage();

await page.goto("http://quotes.toscrape.com/scroll");
await page.waitForSelector(".quote");

for (let i = 0; i < 30; i++) {
  const before = await page.$$eval(".quote", (els) => els.length);
  await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
  try {
    await page.waitForFunction(
      (n) => document.querySelectorAll(".quote").length > n,
      { timeout: 3000 },
      before
    );
  } catch {
    break;
  }
}

const quotes = await page.$$eval(".quote", (els) =>
  els.map((el) => ({
    text: el.querySelector(".text")?.textContent,
    author: el.querySelector(".author")?.textContent,
  }))
);
console.log(quotes.length, quotes[0]);

await page.close();
await browser.disconnect();

import os

from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError

token = os.environ["GOLOGIN_TOKEN"]
profile_id = os.environ["GOLOGIN_PROFILE_ID"]
endpoint = f"wss://cloudbrowser.gologin.com/connect?token={token}&profile={profile_id}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(endpoint)
    page = browser.contexts[0].new_page()

    page.goto("http://quotes.toscrape.com/scroll")
    page.wait_for_selector(".quote")

    for _ in range(30):
        before = page.locator(".quote").count()
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        try:
            page.wait_for_function(
                "n => document.querySelectorAll('.quote').length > n",
                arg=before,
                timeout=3000,
            )
        except PlaywrightTimeoutError:
            break

    quotes = page.eval_on_selector_all(
        ".quote",
        "els => els.map(el => ({text: el.querySelector('.text')?.textContent, author: el.querySelector('.author')?.textContent}))",
    )
    print(len(quotes), quotes[0])

    page.close()
    browser.close()