Blog

import requests from bs4 import BeautifulSoup import time, random headers = {“User-Agent”: “Mozilla/5.0 (compatible; MyBot/1.0)”} session = requests.Session() session.headers.update(headers) def scrape_page(url): resp = session.get(url, timeout=10) resp.raise_for_status() # raises on 4xx / 5xx soup = BeautifulSoup(resp.text, “lxml”) return soup soup = scrape_page(“https://books.toscrape.com/”) books = soup.select(“article.product_pod”) # CSS selector for book in books: title = book.select_one(“h3 a”)[“title”] price = book.select_one(“p.price_color”).text.strip() rating = book.select_one(“p.star-rating”)[“class”][1] print(f“{title} | {price} | {rating}”) time.sleep(random.uniform(0.5, 1.5))

import asyncio, httpx from bs4 import BeautifulSoup async def fetch(client, url): r = await client.get(url, timeout=15) return r.text async def main(urls): async with httpx.AsyncClient() as client: tasks = [fetch(client, u) for u in urls] pages = await asyncio.gather(*tasks) return pages urls = [f“https://example.com/page/{i}” for i in range(1, 6)] pages = asyncio.run(main(urls)) # fetches all 5 concurrently

from playwright.sync_api import sync_playwright with sync_playwright() as p: browser = p.chromium.launch(headless=True) page = browser.new_page() page.goto(“https://spa-site.example.com”) page.wait_for_selector(“table.data-loaded”) # wait for JS render rows = page.query_selector_all(“table tr”) for row in rows: cells = [td.inner_text() for td in row.query_selector_all(“td”)] print(cells) browser.close()

import csv data = [{“title”: “Book A”, “price”: “£9.99”}] # your scraped list with open(“results.csv”, “w”, newline=“”, encoding=“utf-8”) as f: writer = csv.DictWriter(f, fieldnames=[“title”, “price”]) writer.writeheader() writer.writerows(data)

import time, requests def fetch_with_retry(url, retries=3, backoff=2): for attempt in range(retries): try: r = requests.get(url, timeout=10) r.raise_for_status() return r except requests.HTTPError as e: if e.response.status_code == 429: # rate-limited time.sleep(backoff ** attempt) elif e.response.status_code == 404: return None # skip missing pages else: raise except requests.RequestException: time.sleep(backoff ** attempt) return None

from urllib.robotparser import RobotFileParser rp = RobotFileParser() rp.set_url(“https://example.com/robots.txt”) rp.read() if rp.can_fetch(“*”, “https://example.com/products”): print(“Allowed — proceeding”) else: print(“Disallowed by robots.txt — skipping”)