Reddit scraper that bills the thread, not every comment

import html as entities
import json
import os
import re
import time

import requests

thread = "https://www.reddit.com/r/marketing/comments/1x6ko90/why_the_marketing_department_is_about_to_disappear/"


def scrape(url):
    for attempt in range(3):
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": url, "geo": "us", "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if "<shreddit-comment" in page:
            return page
        time.sleep(5)
    raise RuntimeError("No comments came back, try again later")


def attr(tag, name):
    match = re.search(rf'\s{name}="([^"]*)"', tag)
    return entities.unescape(match.group(1)) if match else None


def text(page, element_id):
    start = page.find(f'id="{element_id}"')
    end = page.find("</div>", start)
    paragraphs = re.findall(r"<p>(.*?)</p>", page[start:end], re.S)
    return "\n".join(entities.unescape(re.sub(r"<[^>]+>|\s+", " ", p)).strip() for p in paragraphs)


page = scrape(thread)
post = re.search(r"<shreddit-post\b[^>]*>", page).group(0)
comments = {}
for tag in re.findall(r'<shreddit-comment\b[^>]*thingid="t1_[^>]*>', page):
    comment_id = attr(tag, "thingid")
    comments[comment_id] = {
        "author": attr(tag, "author"),
        "score": attr(tag, "score"),
        "depth": attr(tag, "depth"),
        "text": text(page, f"{comment_id}-comment-rtjson-content"),
    }

print(json.dumps({
    "title": attr(post, "post-title"),
    "score": attr(post, "score"),
    "upvote_ratio": attr(post, "upvote-ratio"),
    "comments_on_reddit": attr(post, "comment-count"),
    "comments": list(comments.values()),
}, indent=2, ensure_ascii=False))

const subreddit = 'SaaS';
const keywords = ['marketing', 'pricing', 'launch'];

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, render_js: 'true', wait: '5000', geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes('<shreddit-post')) return page;
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error('No posts came back, try again later');
}

const page = await scrape(`https://www.reddit.com/r/${subreddit}/new/`);
const attr = (tag, name) => tag.match(new RegExp(`\\s${name}="([^"]*)"`))?.[1]?.replaceAll('&amp;', '&').replaceAll('&#39;', "'").replaceAll('&quot;', '"');

const posts = new Map();
for (const [tag] of page.matchAll(/<shreddit-post\b[^>]*post-title=[^>]*>/g)) {
  posts.set(attr(tag, 'permalink'), {
    title: attr(tag, 'post-title'),
    posted: attr(tag, 'created-timestamp'),
    comments: attr(tag, 'comment-count'),
    url: `https://www.reddit.com${attr(tag, 'permalink')}`,
  });
}

const mentions = [...posts.values()].filter((post) => keywords.some((word) => post.title.toLowerCase().includes(word)));
console.log(`${posts.size} new posts in r/${subreddit}, ${mentions.length} mention your keywords`);
console.table(mentions);

import { writeFileSync } from 'node:fs';

const thread = 'https://www.reddit.com/r/marketing/comments/1x6ko90/why_the_marketing_department_is_about_to_disappear/';

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, geo: 'us', format: 'markdown' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const markdown = response.ok ? await response.text() : '';
    if (markdown.includes('# Comments Section')) return markdown;
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error('No comments came back, try again later');
}

const markdown = await scrape(thread);
const comments = markdown.slice(markdown.indexOf('# Comments Section'));
writeFileSync('thread.md', comments);
console.log(`thread.md: ${Math.round(comments.length / 1024)} KB of comments, ready for your model`);
console.log(comments.slice(0, 600));