Scraping LinkedIn: pagas la página, no la oferta

import html
import json
import os
import re
import time
from urllib.parse import urlencode

import requests

query = "data analyst"
where = "Chicago, IL"


def scrape(url):
    for attempt in range(3):
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": url, "geo": "us", "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if "urn:li:jobPosting:" in page:
            return page
        time.sleep(5)
    raise RuntimeError("No jobs came back, try again later")


def text(pattern, card):
    found = re.search(pattern, card, re.S)
    return " ".join(html.unescape(re.sub(r"<[^>]+>", " ", found.group(1))).split()) if found else None


search = "https://www.linkedin.com/jobs/search?" + urlencode({"keywords": query, "location": where})
page = scrape(search)
jobs = []
for card in page.split("<li")[1:]:
    job_id = re.search(r"urn:li:jobPosting:(\d+)", card)
    if not job_id:
        continue
    jobs.append({
        "title": text(r'base-search-card__title[^>]*>(.*?)</h3>', card),
        "company": text(r'base-search-card__subtitle[^>]*>(.*?)</h4>', card),
        "location": text(r'job-search-card__location[^>]*>(.*?)</span>', card),
        "posted": text(r"<time[^>]*>(.*?)</time>", card),
        "date": text(r'datetime="([^"]+)"', card),
        "url": "https://www.linkedin.com/jobs/view/" + job_id.group(1),
    })

print(json.dumps({"query": query, "where": where, "jobs_on_page": len(jobs), "jobs": jobs}, indent=2, ensure_ascii=False))

import { writeFileSync } from 'node:fs';

const query = 'registered nurse';
const where = 'Texas, United States';

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const body = response.ok ? await response.json() : null;
    if (body?.success) return body.data?.html ?? '';
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error(`Nothing came back for ${url}, try again later`);
}

const text = (card, pattern) =>
  (card.match(pattern)?.[1] ?? '').replace(/<[^>]+>/g, ' ').replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/\s+/g, ' ').trim();
const cell = (value) => `"${value.replace(/"/g, '""')}"`;

const jobs = new Map();
let pages = 0;

for (let start = 0; start < 1000; start += 10) {
  const params = new URLSearchParams({ keywords: query, location: where, start });
  const page = await scrape(`https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?${params}`);
  pages += 1;
  const cards = page.split('<li').slice(1).filter((card) => card.includes('urn:li:jobPosting:'));
  if (cards.length === 0) break;
  for (const card of cards) {
    const id = card.match(/urn:li:jobPosting:(\d+)/)[1];
    jobs.set(id, [
      text(card, /base-search-card__title[^>]*>([\s\S]*?)<\/h3>/),
      text(card, /base-search-card__subtitle[^>]*>([\s\S]*?)<\/h4>/),
      text(card, /job-search-card__location[^>]*>([\s\S]*?)<\/span>/),
      card.match(/datetime="([^"]+)"/)?.[1] ?? '',
      `https://www.linkedin.com/jobs/view/${id}`,
    ]);
  }
}

const rows = [['title', 'company', 'location', 'date', 'url'], ...jobs.values()];
writeFileSync('linkedin-jobs.csv', rows.map((row) => row.map(cell).join(',')).join('\n'));
console.log(`${jobs.size} jobs from ${pages} pages saved to linkedin-jobs.csv`);
console.log(rows.slice(1, 4).map((row) => row.slice(0, 3).join(' | ')).join('\n'));

const query = 'software engineer';
const city = 'Austin, TX';

async function scrape(url, marker) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const params = new URLSearchParams({ url, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes(marker)) return page;
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
  throw new Error(`Nothing came back for ${url}, try again later`);
}

const text = (markup = '') => markup.replace(/<[^>]+>/g, ' ').replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/\s+/g, ' ').trim();
const key = (job) => `${job.company} ${job.title}`.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();

const linkedin = await scrape(`https://www.linkedin.com/jobs/search?${new URLSearchParams({ keywords: query, location: city })}`, 'urn:li:jobPosting:');
const fromLinkedin = linkedin
  .split('<li')
  .filter((card) => card.includes('urn:li:jobPosting:'))
  .map((card) => ({
    board: 'LinkedIn',
    title: text(card.match(/base-search-card__title[^>]*>([\s\S]*?)<\/h3>/)?.[1]),
    company: text(card.match(/base-search-card__subtitle[^>]*>([\s\S]*?)<\/h4>/)?.[1]),
    location: text(card.match(/job-search-card__location[^>]*>([\s\S]*?)<\/span>/)?.[1]),
    url: `https://www.linkedin.com/jobs/view/${card.match(/urn:li:jobPosting:(\d+)/)[1]}`,
  }));

const indeed = await scrape(`https://www.indeed.com/jobs?${new URLSearchParams({ q: query, l: city, limit: 50 })}`, 'mosaic-provider-jobcards"]=');
const start = indeed.indexOf('=', indeed.indexOf('mosaic-provider-jobcards"]=')) + 1;
const cards = JSON.parse(indeed.slice(start, indeed.indexOf(';\n', start)));
const results = cards.metaData.mosaicProviderJobCardsModel.results;
const fromIndeed = results.filter((job, i) => results.findIndex((other) => other.jobkey === job.jobkey) === i).map((job) => ({
  board: 'Indeed',
  title: job.title,
  company: job.company,
  location: job.formattedLocation,
  url: `https://www.indeed.com/viewjob?jk=${job.jobkey}`,
}));

const jobs = new Map();
for (const job of [...fromLinkedin, ...fromIndeed]) {
  const seen = jobs.get(key(job));
  if (seen) seen.boards.add(job.board);
  else jobs.set(key(job), { ...job, boards: new Set([job.board]) });
}

const onBoth = [...jobs.values()].filter((job) => job.boards.size === 2);
console.log(`LinkedIn ${fromLinkedin.length} + Indeed ${fromIndeed.length} = ${jobs.size} different jobs, ${onBoth.length} on both boards, 2 pages billed`);
for (const job of onBoth.slice(0, 5)) console.log(`${job.title} | ${job.company} | ${job.location}`);