Instagram スクレイピング：​ログイン不要で、​フォロワー数も​正確

import json
import os
import re
import time

import requests

username = "nasa"


def scrape(url, marker):
    for attempt in range(3):
        address = url if attempt == 0 else f"{url}?fresh={int(time.time())}"
        response = requests.get(
            "https://parsing.webunlocker.gologin.com/v1/scrape",
            params={"url": address, "geo": "us", "format": "json"},
            headers={"apikey": os.environ["GOLOGIN_SCRAPING_API_KEY"]},
            timeout=90,
        )
        page = response.json().get("data", {}).get("html", "") if response.ok else ""
        if marker in page:
            return page
        time.sleep(3)
    raise RuntimeError("The page came back without data, try again later")


def blocks(page):
    for block in re.findall(r'<script type="application/json"[^>]*>(.*?)</script>', page, re.S):
        try:
            yield json.loads(block)
        except json.JSONDecodeError:
            pass


def find(node, key):
    if isinstance(node, dict):
        for name, value in node.items():
            if name == key:
                yield value
            yield from find(value, key)
    elif isinstance(node, list):
        for item in node:
            yield from find(item, key)


data = list(blocks(scrape(f"https://www.instagram.com/{username}/", '"follower_count"')))
user = next(item for item in find(data, "xig_user_by_username") if "follower_count" in item)

posts = {}
for timeline in find(data, "polaris_ordered_timeline_connection"):
    for edge in timeline["edges"]:
        post = edge["node"]
        reel = post.get("product_type") == "clips"
        posts[post["code"]] = {
            "url": f"https://www.instagram.com/{'reel' if reel else 'p'}/{post['code']}/",
            "type": "reel" if reel else "post",
            "caption": (post.get("caption") or {}).get("text", "").split("\n")[0],
            "image": post.get("display_uri"),
        }

profile = {
    "username": user["username"],
    "full_name": user["full_name"],
    "biography": user["biography"],
    "verified": user["is_verified"],
    "followers": user["follower_count"],
    "following": user["following_count"],
    "latest_posts": list(posts.values())[:12],
}

print(json.dumps(profile, indent=2, ensure_ascii=False))
print(f"@{username}: {profile['followers']:,} followers, {len(profile['latest_posts'])} latest posts from one page")

import { writeFileSync } from 'node:fs';

const username = 'natgeo';

async function scrape(url, marker) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}?fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes(marker)) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  throw new Error(`${url} came back without data, try again later`);
}

function* find(node, test) {
  if (Array.isArray(node)) for (const item of node) yield* find(item, test);
  else if (node && typeof node === 'object') {
    if (test(node)) yield node;
    for (const value of Object.values(node)) yield* find(value, test);
  }
}

const data = (page) =>
  [...page.matchAll(/<script type="application\/json"[^>]*>([\s\S]*?)<\/script>/g)].flatMap(([, json]) => {
    try {
      return [JSON.parse(json)];
    } catch {
      return [];
    }
  });

const profile = data(await scrape(`https://www.instagram.com/${username}/`, '"follower_count"'));
const user = find(profile, (node) => typeof node.follower_count === 'number').next().value;
const codes = [...new Set([...find(profile, (node) => node.code && node.display_uri)].map((post) => post.code))].slice(0, 12);

const posts = [];
for (const code of codes) {
  const page = data(await scrape(`https://www.instagram.com/p/${code}/`, '"like_count"'));
  const post = find(page, (node) => typeof node.like_count === 'number' && node.code === code).next().value;
  posts.push({ url: `https://www.instagram.com/p/${code}/`, date: new Date(post.taken_at * 1000).toISOString().slice(0, 10), likes: post.like_count, comments: post.comment_count });
}

const csv = ['url,date,likes,comments', ...posts.map((post) => `${post.url},${post.date},${post.likes},${post.comments}`)].join('\n');
writeFileSync(`${username}-posts.csv`, csv);

const average = posts.reduce((sum, post) => sum + post.likes + post.comments, 0) / posts.length;
console.table(posts);
console.log(`@${username}: ${user.follower_count.toLocaleString('en-US')} followers, ${Math.round(average).toLocaleString('en-US')} likes and comments a post`);
console.log(`Engagement rate ${((average / user.follower_count) * 100).toFixed(2)}% from ${posts.length + 1} pages: $${((posts.length + 1) * 0.00098).toFixed(4)} on the Growth plan`);

import { writeFileSync } from 'node:fs';

const usernames = ['nasa', 'natgeo', 'nike', 'spotify', 'starbucks', 'redbull'];

async function scrape(url) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    const address = attempt === 1 ? url : `${url}?fresh=${Date.now()}`;
    const params = new URLSearchParams({ url: address, geo: 'us', format: 'json' });
    const response = await fetch(`https://parsing.webunlocker.gologin.com/v1/scrape?${params}`, {
      headers: { apikey: process.env.GOLOGIN_SCRAPING_API_KEY },
    });
    const page = response.ok ? (await response.json()).data?.html : '';
    if (page?.includes('"follower_count"')) return page;
    await new Promise((resolve) => setTimeout(resolve, 3000));
  }
  return '';
}

const field = (page, name) => page.match(new RegExp(`"${name}":("(?:[^"\\\\]|\\\\.)*"|\\d+|true|false)`))?.[1];

const rows = [];
for (const username of usernames) {
  const page = await scrape(`https://www.instagram.com/${username}/`);
  rows.push({
    username,
    name: page && JSON.parse(field(page, 'full_name')),
    verified: page && field(page, 'is_verified') === 'true',
    followers: page && Number(field(page, 'follower_count')),
    following: page && Number(field(page, 'following_count')),
    posts: page.match(/([\d.,]+[KMB]?) Posts/)?.[1],
  });
}

writeFileSync('profiles.csv', ['username,name,verified,followers,following,posts', ...rows.map((row) => Object.values(row).map((value) => JSON.stringify(value ?? '')).join(','))].join('\n'));
console.table(rows);
console.log(`${rows.length} profiles in ${rows.length} pages: $${(rows.length * 0.00098).toFixed(4)} on the Growth plan`);