Facebook 爬虫，​采集公共主页和帖子，​无需登录

import html
import json
import os
import re
import time
from datetime import datetime, timezone

import requests

page_url = "https://www.facebook.com/NASA/"


def scrape(url, marker):
    for attempt in range(3):
        address = url if attempt == 0 else f"{url}?fresh={int(time.time())}"
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": address, "geo": "us", "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if marker in page:
            return page
        time.sleep(3)
    raise RuntimeError("The page came back without data, try again later")


def blocks(page):
    for block in re.findall(r'<script type="application/json"[^>]*>(.*?)</script>', page, re.S):
        try:
            yield json.loads(block)
        except json.JSONDecodeError:
            pass


def find(node, key):
    if isinstance(node, dict):
        for name, value in node.items():
            if name == key:
                yield value
            yield from find(value, key)
    elif isinstance(node, list):
        for item in node:
            yield from find(item, key)


def first(node, key, field):
    return next(item for item in find(node, key) if isinstance(item, dict) and field in item)


page = scrape(page_url, "talking about this")
data = list(blocks(page))
meta = dict(re.findall(r'<meta property="(og:\w+)" content="([^"]*)"', page))
about = html.unescape(meta["og:description"])
count = lambda words: int(re.search(rf"([\d,]+) {words}", about).group(1).replace(",", ""))

details = first(data, "delegate_page", "category_name")
story = first(data, "timeline_list_feed_units", "edges")["edges"][0]["node"]
feedback = first(story, "feedback", "reaction_count")

facebook_page = {
    "name": html.unescape(meta["og:title"]),
    "followers": count("followers"),
    "talking_about_this": count("talking about this"),
    "category": details["category_name"],
    "intro": details["best_description"]["text"],
    "websites": [
        item["renderer"]["context_item"]["plaintext_title"]["text"]
        for item in find(data, "timeline_context_item")
        if item["renderer"]["__typename"] == "WebsiteContextItemRenderer"
    ],
    "latest_post": {
        "url": next(find(story, "permalink_url")),
        "date": datetime.fromtimestamp(story["creation_time"], timezone.utc).isoformat(),
        "text": first(story, "message", "text")["text"],
        "reactions": feedback["reaction_count"]["count"],
        "comments": first(story, "comments", "total_count")["total_count"],
        "shares": feedback["share_count"]["count"],
    },
}

print(json.dumps(facebook_page, indent=2, ensure_ascii=False))
print(f"{facebook_page['name']}: {facebook_page['followers']:,} followers and the latest post from one page")

import { writeFileSync } from 'node:fs';

const posts = [
  'https://www.facebook.com/NASA/posts/pfbid05YJTkWebVNj83FA2ifGfHBxTdYFWuNrAMo4yAKnbRuY9XXK8nXohgsLxADToV4vfl',
  'https://www.facebook.com/Starbucks/posts/pfbid02v2F2jafXiaaRi3PAramXqszA3W5rRcJtaF8MXE1s453xiHbK9JJNmMqnuzV5i8Gxl',
  'https://www.facebook.com/McDonalds/posts/pfbid04kS5YA29jgtMNfHTQCf8JthvEM71jgVJzQN6L53wU7ck5ocPBY49sdFUcmt6L8Nsl',
  'https://www.facebook.com/walmart/posts/pfbid02Tcu3cY8cHDhE43MsF7qSQnUDpvMZux6P7FfFsWswowv5QJTVKMeDou73PW6M5c22l',
  'https://www.facebook.com/Spotify/posts/pfbid0WmiXu2TVoUdrNDBEJepHYbrukgikHR6euuvJGuLiVNtvTbXkXFywG5rEKCZcRijhl',
  'https://www.facebook.com/adidas/posts/pfbid02zsJxukrC4EagRJPwg958TiWViTm9NSCPETubYgPmazax2DKqhZwaNCHBHHaXntvl',
];

const embed = (post) => `https://www.facebook.com/plugins/post.php?${new URLSearchParams({ href: post, show_text: 'true', width: '500' })}`;

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}&fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes('data-testid="post_message"')) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return '';
}

const count = (page, name) => page.match(new RegExp(`sx_post-plugin_${name}"></i>([^<]+)<`))?.[1] ?? '';
const plain = (html = '') =>
  html
    .replace(/<\/p>\s*<p>|<br\s*\/?>/g, ' ')
    .replace(/<[^>]+>/g, '')
    .replace(/&amp;/g, '&')
    .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(code))
    .replace(/\s+/g, ' ')
    .trim();

const rows = [];
for (const post of posts) {
  const page = await scrape(embed(post));
  const posted = page.match(/data-utime="(\d+)"/)?.[1];
  rows.push({
    page: plain(page.match(/<span class="_2_79 _50f7">([^<]+)<|<a title="([^"]+)" class="_5pb8|aria-label="([^"]+)" role="img"/)?.slice(1).find(Boolean)),
    date: posted ? new Date(posted * 1000).toISOString().slice(0, 10) : '',
    likes: count(page, 'like'),
    comments: count(page, 'comment'),
    shares: count(page, 'share'),
    text: plain(page.match(/data-testid="post_message"[^>]*>([\s\S]*?)<\/div>/)?.[1]).slice(0, 80),
  });
}

writeFileSync('posts.csv', [Object.keys(rows[0]).join(','), ...rows.map((row) => Object.values(row).map((value) => JSON.stringify(value)).join(','))].join('\n'));
console.table(rows);
console.log(`${rows.length} posts in ${rows.length} requests: $${(rows.length * 0.00098).toFixed(4)} on the Growth plan`);

import { writeFileSync } from 'node:fs';

const pages = ['NASA', 'Starbucks', 'nike', 'netflix', 'redbull', 'LEGO'];

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}?fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes('talking about this')) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return '';
}

const number = (text) => (text ? Number(text.replaceAll(',', '')) : '');

const rows = [];
for (const name of pages) {
  const page = await scrape(`https://www.facebook.com/${name}/`);
  const about = page.match(/<meta property="og:description" content="([^"]*)"/)?.[1] ?? '';
  const feed = page.slice(page.indexOf('"timeline_list_feed_units"'));
  const posted = feed.match(/"creation_time":(\d+)/)?.[1];
  rows.push({
    page: name,
    followers: number(about.match(/([\d,]+) followers/)?.[1]),
    talking: number(about.match(/([\d,]+) talking about this/)?.[1]),
    category: page.match(/"category_name":"([^"]+)"/)?.[1].replaceAll('\\/', '/') ?? '',
    posted: posted ? new Date(posted * 1000).toISOString().slice(0, 10) : '',
    reactions: number(feed.match(/"reaction_count":\{"count":(\d+)/)?.[1]),
    comments: number(feed.match(/"comments":\{"total_count":(\d+)/)?.[1]),
    shares: number(feed.match(/"share_count":\{"count":(\d+)/)?.[1]),
  });
}

writeFileSync('pages.csv', [Object.keys(rows[0]).join(','), ...rows.map((row) => Object.values(row).map((value) => JSON.stringify(value)).join(','))].join('\n'));
console.table(rows);
console.log(`${rows.length} brand pages in ${rows.length} requests: $${(rows.length * 0.00098).toFixed(4)} on the Growth plan`);