Indeed 爬虫，每页返回 50 个职位

import json
import os
import time
from urllib.parse import urlencode

import requests

query = "marketing manager"
where = "Austin, TX"


def scrape(url):
    for attempt in range(3):
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": url, "geo": "us", "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if '"results":[{' in page:
            return page
        time.sleep(5)
    raise RuntimeError("No jobs came back, try again later")


def embedded(page, marker):
    start = page.index(marker) + len(marker) - 1
    value, _ = json.JSONDecoder().raw_decode(page[start:])
    return value


search = "https://www.indeed.com/jobs?" + urlencode({"q": query, "l": where, "limit": 50})
page = scrape(search)
jobs = [
    {
        "title": job["title"],
        "company": job["company"],
        "location": job["formattedLocation"],
        "salary": (job.get("salarySnippet") or {}).get("text"),
        "rating": job.get("companyRating") or None,
        "posted": job.get("formattedRelativeTime"),
        "sponsored": job.get("sponsored", False),
        "url": "https://www.indeed.com/viewjob?jk=" + job["jobkey"],
    }
    for job in embedded(page, '"results":[')
]

print(json.dumps({"query": query, "where": where, "jobs_on_page": len(jobs), "jobs": jobs}, indent=2, ensure_ascii=False))

const search = 'https://www.indeed.com/jobs?q=registered+nurse&l=Remote&limit=50';
const wanted = 5;

async function scrape(url, marker) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes(marker)) return page;
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error(`Nothing came back for ${url}, try again later`);
}

const field = (page, name) => JSON.parse(page.match(new RegExp(`"${name}":("(?:[^"\\\\]|\\\\.)*")`))?.[1] ?? '""');
const text = (markup) => markup.replace(/<[^>]+>/g, ' ').replace(/&amp;/g, '&').replace(/&nbsp;/g, ' ').replace(/\s+/g, ' ').trim();

const listing = await scrape(search, '"results":[{');
const keys = [...new Set([...listing.matchAll(/"jobkey":"([0-9a-f]{16})"/g)].map((match) => match[1]))];
console.log(`${keys.length} jobs on the search page, reading the first ${wanted} in full`);

for (const key of keys.slice(0, wanted)) {
  const job = await scrape(`https://www.indeed.com/m/viewjob?jk=${key}`, '"sanitizedJobDescription"');
  const description = text(field(job, 'sanitizedJobDescription'));
  console.log(`\n${field(job, 'jobTitle')} at ${field(job, 'companyName')}, ${field(job, 'formattedLocation')} (${description.length} characters)`);
  console.log(`${description.slice(0, 280)}...`);
}

const company = 'Amazon.com';
const pages = 5;

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes('"results":[{')) return page;
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error(`No jobs came back for ${url}, try again later`);
}

function embedded(page, marker) {
  const start = page.indexOf(marker) + marker.length - 1;
  const token = /"(?:[^"\\]|\\.)*"|[[\]{}]/g;
  token.lastIndex = start;
  let depth = 0;
  for (let match = token.exec(page); match; match = token.exec(page)) {
    depth += '[{'.includes(match[0]) ? 1 : ']}'.includes(match[0]) ? -1 : 0;
    if (depth === 0) return JSON.parse(page.slice(start, match.index + 1));
  }
  return [];
}

const jobs = new Map();
for (let start = 0; start < pages * 20; start += 20) {
  const page = await scrape(`https://www.indeed.com/cmp/${company}/jobs?start=${start}`);
  const fresh = embedded(page, '"results":[').filter((job) => !jobs.has(job.jobkey));
  if (fresh.length === 0) break;
  for (const job of fresh) {
    jobs.set(job.jobkey, { title: job.title, location: job.formattedLocation, salary: job.salarySnippet?.text ?? null, posted: job.formattedRelativeTime });
  }
  console.log(`page ${start / 20 + 1}: ${fresh.length} new jobs, ${jobs.size} so far`);
}

console.table([...jobs.values()].slice(0, 10));