TikTok scraper that vets a creator in one request

import html
import json
import os
import re
import time

import requests

username = "mrbeast"


def scrape(url, marker):
    for attempt in range(3):
        address = url if attempt == 0 else f"{url}?fresh={int(time.time())}"
        try:
            response = requests.get(
                "https://parsing.webunlocker.gologin.com/v1/scrape",
                params={"url": address, "geo": "us", "format": "json"},
                headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
                timeout=60,
            )
            page = response.json().get("data", {}).get("html", "") if response.ok else ""
        except requests.RequestException:
            page = ""
        if marker in page:
            return page
        time.sleep(3)
    raise RuntimeError("TikTok did not send the page, try again later")


page = scrape(f"https://www.tiktok.com/@{username}", '"followerCount"')
script = re.search(r'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)</script>', page, re.S)
info = json.loads(script.group(1))["__DEFAULT_SCOPE__"]["webapp.user-detail"]["userInfo"]
user, stats = info["user"], info["statsV2"]

videos = []
for item in page.split('data-e2e="user-post-item"')[1:]:
    video = re.search(r'/video/(\d+)"', item)
    views = re.search(r'data-e2e="video-views"[^>]*>([^<]*)<', item)
    caption = re.search(r'<img alt="([^"]*)"', item)
    if video:
        videos.append({
            "url": f"https://www.tiktok.com/@{username}/video/{video.group(1)}",
            "views": views.group(1) if views else None,
            "caption": html.unescape(caption.group(1)).split(" created by ")[0].strip() if caption else "",
        })

profile = {
    "username": user["uniqueId"],
    "nickname": user["nickname"],
    "bio": user["signature"],
    "verified": user["verified"],
    "followers": int(stats["followerCount"]),
    "following": int(stats["followingCount"]),
    "likes": int(stats["heartCount"]),
    "videos": int(stats["videoCount"]),
    "latest_videos": videos,
}

print(json.dumps(profile, indent=2, ensure_ascii=False))
print(f"@{username}: {profile['followers']:,} followers, {len(videos)} latest videos from one page")

import { writeFileSync } from 'node:fs';

const username = 'mrbeast';

async function scrape(url, marker) {
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const api = new URL('https://parsing.webunlocker.gologin.com/v1/scrape');
    api.search = new URLSearchParams({ url: attempt === 0 ? url : `${url}?fresh=${Date.now()}`, geo: 'us', format: 'json' });
    try {
      const response = await fetch(api, {
        headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
        signal: AbortSignal.timeout(60_000),
      });
      const page = response.ok ? ((await response.json()).data?.html ?? '') : '';
      if (page.includes(marker)) return page;
    } catch {}
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  throw new Error(`TikTok did not send ${url}, try again later`);
}

const data = (page) =>
  JSON.parse(page.match(/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/)[1]).__DEFAULT_SCOPE__;

const profile = await scrape(`https://www.tiktok.com/@${username}`, '"followerCount"');
const followers = Number(data(profile)['webapp.user-detail'].userInfo.statsV2.followerCount);
const links = profile
  .split('data-e2e="user-post-item"')
  .slice(1)
  .map((item) => item.match(/\/video\/(\d+)"/)?.[1])
  .filter(Boolean)
  .map((id) => `https://www.tiktok.com/@${username}/video/${id}`);

const rows = [];
for (const url of links) {
  const video = data(await scrape(url, '"playCount"'))['webapp.video-detail'].itemInfo.itemStruct;
  const stats = video.statsV2;
  rows.push({
    url,
    date: new Date(video.createTime * 1000).toISOString().slice(0, 10),
    plays: Number(stats.playCount),
    likes: Number(stats.diggCount),
    comments: Number(stats.commentCount),
    shares: Number(stats.shareCount),
    saves: Number(stats.collectCount),
    caption: video.desc,
  });
}

const cell = (value) => `"${String(value).replaceAll('"', '""')}"`;
writeFileSync(
  `${username}-videos.csv`,
  [Object.keys(rows[0]).join(','), ...rows.map((row) => Object.values(row).map(cell).join(','))].join('\n'),
);

const sum = (key) => rows.reduce((total, row) => total + row[key], 0);
const engagement = (sum('likes') + sum('comments') + sum('shares') + sum('saves')) / sum('plays');
console.table(rows.map(({ caption, ...row }) => row));
console.log(
  `@${username}: ${followers.toLocaleString('en')} followers, ${rows.length} videos, ${(engagement * 100).toFixed(1)}% engagement, ${rows.length + 1} pages, saved to ${username}-videos.csv`,
);

import { writeFileSync } from 'node:fs';

const creators = ['mrbeast', 'nasa', 'khaby.lame', 'zachking', 'charlidamelio'];

async function scrape(url, marker) {
  for (let attempt = 0; attempt < 3; attempt += 1) {
    const api = new URL('https://parsing.webunlocker.gologin.com/v1/scrape');
    api.search = new URLSearchParams({ url: attempt === 0 ? url : `${url}?fresh=${Date.now()}`, geo: 'us', format: 'json' });
    try {
      const response = await fetch(api, {
        headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
        signal: AbortSignal.timeout(60_000),
      });
      const page = response.ok ? ((await response.json()).data?.html ?? '') : '';
      if (page.includes(marker)) return page;
    } catch {}
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return null;
}

const UNITS = { K: 1e3, M: 1e6, B: 1e9 };
const number = (text) => Number.parseFloat(text) * (UNITS[text.at(-1)] ?? 1);

const rows = [];
for (const username of creators) {
  const page = await scrape(`https://www.tiktok.com/@${username}`, '"followerCount"');
  if (!page) {
    console.log(`@${username}: no page this time, run it again later`);
    continue;
  }
  const script = page.match(/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/)[1];
  const { user, statsV2 } = JSON.parse(script).__DEFAULT_SCOPE__['webapp.user-detail'].userInfo;
  const views = [...page.matchAll(/data-e2e="video-views"[^>]*>([^<]+)</g)].map(([, text]) => number(text)).sort((a, b) => a - b);
  rows.push({
    username: user.uniqueId,
    name: user.nickname,
    verified: user.verified,
    followers: Number(statsV2.followerCount),
    likes: Number(statsV2.heartCount),
    videos: Number(statsV2.videoCount),
    median_views: views.length ? views[Math.floor(views.length / 2)] : '',
  });
}

const cell = (value) => `"${String(value).replaceAll('"', '""')}"`;
writeFileSync('creators.csv', [Object.keys(rows[0]).join(','), ...rows.map((row) => Object.values(row).map(cell).join(','))].join('\n'));
console.table(rows);
console.log(`${rows.length} of ${creators.length} creators, one page each, saved to creators.csv`);