Tutorial 9: Scrapy + BeautifulSoup - Data Collection for ML

Table of Contents

Introduction

Machine learning models are only as good as the data they are trained on. While many curated datasets exist, real-world ML projects often require collecting custom data from the web. Whether you need product reviews for sentiment analysis, news articles for text classification, or job listings for market analysis, web scraping is an essential skill for ML practitioners.

This tutorial covers two of the most powerful Python tools for web scraping: BeautifulSoup for parsing HTML and Scrapy for building scalable, production-grade web crawlers. You will learn how to extract structured data from websites, clean it, store it in various formats, and transform it into ML-ready datasets, all while respecting ethical scraping guidelines.

Prerequisites

• Python 3.9 or higher
• Basic understanding of HTML structure and CSS selectors
• Familiarity with Python data structures

Install the required packages:

pip install scrapy beautifulsoup4 lxml requests pandas sqlalchemy
pip install scrapy-playwright  # For JavaScript-rendered pages

BeautifulSoup Basics

Parsing HTML Documents

from bs4 import BeautifulSoup
import requests

Fetch a web page
url = "https://quotes.toscrape.com/"
response = requests.get(url)
soup = BeautifulSoup(response.text, "lxml")

Print the page title
print(f"Page title: {soup.title.string}")

Navigating the DOM Tree

# Find elements by tag
alldivs = soup.findall("div")
print(f"Found {len(alldivs)} div elements")


Find by class name
quotes = soup.findall("div", class="quote")

print(f"Found {len(quotes)} quotes on the page")

Find by ID
specific = soup.find(id="main-content")

Find by CSS selector
authors = soup.select("small.author")
for author in authors:
    print(f"Author: {author.gettext()}")

Extracting Data

# Extract structured data from each quote
data = []
for quote in soup.findall("div", class="quote"):
    text = quote.find("span", class="text").gettext()
    author = quote.find("small", class="author").gettext()
    tags = [tag.gettext() for tag in quote.findall("a", class="tag")]


    data.append({
        "text": text,
        "author": author,
        "tags": tags,
    })

for item in data[:3]:
    print(f'"{item["text"][:60]}..." - {item["author"]}')
    print(f"  Tags: {', '.join(item['tags'])}")

# Extract all links from a page
links = []
for atag in soup.findall("a", href=True):

    href = atag["href"]
    linktext = atag.gettext(strip=True)

    links.append({"url": href, "text": linktext})

Extract image sources
images = []
for img in soup.findall("img"):

    src = img.get("src", "")
    alt = img.get("alt", "No alt text")
    images.append({"src": src, "alt": alt})

Extract data attributes
elements = soup.findall(attrs={"data-category": True})
for el in elements:
    print(f"Category: {el['data-category']}")

Scrapy Fundamentals

Creating a Scrapy Project

# Create a new Scrapy project
scrapy startproject mldatacollector
cd mldatacollector

Generate a spider
scrapy genspider quotesspider quotes.toscrape.com

mldatacollector/
    scrapy.cfg
    mldatacollector/

        init.py
        items.py          # Data models
        middlewares.py     # Request/response processing
        pipelines.py      # Data processing pipelines
        settings.py       # Project configuration
        spiders/
            init.py
            quotesspider.py

Building Scrapy Spiders

Basic Spider

# mldatacollector/spiders/quotesspider.py
import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    alloweddomains = ["quotes.toscrape.com"]
    starturls = ["https://quotes.toscrape.com/"]


    def parse(self, response):
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
                "tags": quote.css("a.tag::text").getall(),
                "authorurl": response.urljoin(
                    quote.css("a[href='author']::attr(href)").get("")

                ),
            }

        # Follow pagination
        nextpage = response.css("li.next a::attr(href)").get()

        if nextpage:
            yield response.follow(nextpage, callback=self.parse)

# mldatacollector/spiders/newsspider.py
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class NewsSpider(CrawlSpider):
    name = "news"
    alloweddomains = ["example-news.com"]

    starturls = ["https://example-news.com/"]

    rules = (
        # Follow category links
        Rule(LinkExtractor(allow=r"/category/\w+"), follow=True),
        # Extract data from article pages
        Rule(
            LinkExtractor(allow=r"/article/\d+"),
            callback="parsearticle",

            follow=False,
        ),
    )

    def parsearticle(self, response):
        yield {
            "title": response.css("h1.article-title::text").get("").strip(),
            "content": " ".join(
                response.css("div.article-body p::text").getall()
            ),
            "author": response.css("span.author-name::text").get("").strip(),
            "publisheddate": response.css(

                "time.published::attr(datetime)"
            ).get(""),
            "category": response.css("a.category-link::text").get("").strip(),
            "tags": response.css("a.article-tag::text").getall(),
            "url": response.url,
        }

# mldatacollector/spiders/authspider.py
import scrapy

class AuthenticatedSpider(scrapy.Spider):
    name = "authspider"

    starturls = ["https://example.com/login"]

    def parse(self, response):
        # Extract CSRF token if present
        csrftoken = response.css("input[name='csrftoken']::attr(value)").get("")

        return scrapy.FormRequest.fromresponse(

            response,
            formdata={
                "username": "yourusername",
                "password": "yourpassword",

                "csrftoken": csrftoken,

            },
            callback=self.afterlogin,
        )

    def afterlogin(self, response):

        if "Login failed" in response.text:
            self.logger.error("Login failed!")
            return

        # Now crawl authenticated pages
        yield response.follow("/dashboard/data", callback=self.parsedata)

    def parsedata(self, response):

        for row in response.css("table.data-table tr"):
            cells = row.css("td::text").getall()
            if cells:
                yield {
                    "column1": cells[0].strip() if len(cells) > 0 else "",
                    "column2": cells[1].strip() if len(cells) > 1 else "",

                    "column3": cells[2].strip() if len(cells) > 2 else "",
                }

# mldatacollector/items.py
import scrapy

class ArticleItem(scrapy.Item):
    title = scrapy.Field()
    content = scrapy.Field()
    author = scrapy.Field()
    publisheddate = scrapy.Field()

    category = scrapy.Field()
    tags = scrapy.Field()
    url = scrapy.Field()
    wordcount = scrapy.Field()
    scrapedat = scrapy.Field()


class ProductReviewItem(scrapy.Item):
    productname = scrapy.Field()
    reviewtext = scrapy.Field()

    rating = scrapy.Field()
    reviewer = scrapy.Field()
    reviewdate = scrapy.Field()
    verifiedpurchase = scrapy.Field()

    helpfulvotes = scrapy.Field()

# mldatacollector/pipelines.py
import re
from datetime import datetime
from itemadapter import ItemAdapter

class CleanTextPipeline:
    """Clean and normalize text fields."""

    def processitem(self, item, spider):

        adapter = ItemAdapter(item)

        # Clean text fields
        for field in ["title", "content", "reviewtext"]:
            if adapter.get(field):
                text = adapter[field]
                text = re.sub(r"\s+", " ", text)       # Normalize whitespace
                text = text.strip()
                text = re.sub(r"[^\x00-\x7F]+", "", text)  # Remove non-ASCII
                adapter[field] = text

        return item

class DeduplicationPipeline:
    """Remove duplicate items based on URL or content."""

    def init(self):
        self.seenurls = set()

        self.seencontenthashes = set()


    def processitem(self, item, spider):
        adapter = ItemAdapter(item)
        import hashlib

        # Check URL duplication
        url = adapter.get("url", "")
        if url and url in self.seenurls:

            from scrapy.exceptions import DropItem
            raise DropItem(f"Duplicate URL: {url}")
        self.seenurls.add(url)

        # Check content duplication
        content = adapter.get("content", "") or adapter.get("reviewtext", "")

        if content:
            contenthash = hashlib.md5(content.encode()).hexdigest()
            if contenthash in self.seencontenthashes:

                from scrapy.exceptions import DropItem
                raise DropItem("Duplicate content detected")
            self.seencontenthashes.add(contenthash)

        return item

class AddMetadataPipeline:
    """Add metadata to each item."""

    def processitem(self, item, spider):

        adapter = ItemAdapter(item)
        adapter["scrapedat"] = datetime.utcnow().isoformat()

        # Calculate word count
        content = adapter.get("content", "") or adapter.get("reviewtext", "")

        if content:
            adapter["wordcount"] = len(content.split())

        return item

class FilterPipeline:
    """Filter out items that do not meet quality criteria."""

    MINCONTENTLENGTH = 100

    def processitem(self, item, spider):

        adapter = ItemAdapter(item)
        content = adapter.get("content", "") or adapter.get("reviewtext", "")

        if not content or len(content) < self.MINCONTENTLENGTH:
            from scrapy.exceptions import DropItem
            raise DropItem(f"Content too short: {len(content)} chars")

        return item

# mldatacollector/middlewares.py
import random

class RotateUserAgentMiddleware:
    """Rotate User-Agent headers to avoid detection."""

    USERAGENTS = [

        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10157) AppleWebKit/605.1.15 "

        "(KHTML, like Gecko) Version/17.0 Safari/605.1.15",
        "Mozilla/5.0 (X11; Linux x8664; rv:121.0) Gecko/20100101 Firefox/121.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) "
        "Gecko/20100101 Firefox/121.0",
    ]

    def processrequest(self, request, spider):

        request.headers["User-Agent"] = random.choice(self.USERAGENTS)

class RetryMiddleware:
    """Custom retry logic with exponential backoff."""

    MAXRETRIES = 3


    def processresponse(self, request, response, spider):
        if response.status in [429, 500, 502, 503, 504]:
            retries = request.meta.get("retrycount", 0)

            if retries < self.MAXRETRIES:
                import time
                waittime = 2  retries

                spider.logger.warning(
                    f"Retrying {request.url} (attempt {retries + 1}), "
                    f"waiting {waittime}s"
                )
                time.sleep(waittime)

                request.meta["retrycount"] = retries + 1
                return request
        return response

# mldatacollector/settings.py

BOTNAME = "mldatacollector"

SPIDERMODULES = ["mldatacollector.spiders"]
NEWSPIDERMODULE = "mldatacollector.spiders"


Obey robots.txt
ROBOTSTXTOBEY = True

Rate limiting
CONCURRENTREQUESTS = 8

DOWNLOADDELAY = 1.5
CONCURRENTREQUESTSPERDOMAIN = 4

RANDOMIZEDOWNLOADDELAY = True


Pipeline configuration (lower number = higher priority)
ITEMPIPELINES = {
    "mldatacollector.pipelines.CleanTextPipeline": 100,
    "mldatacollector.pipelines.FilterPipeline": 200,
    "mldatacollector.pipelines.DeduplicationPipeline": 300,
    "mldatacollector.pipelines.AddMetadataPipeline": 400,
}

Middleware configuration
DOWNLOADERMIDDLEWARES = {

    "mldatacollector.middlewares.RotateUserAgentMiddleware": 400,

    "mldatacollector.middlewares.RetryMiddleware": 500,

}

Cache for development
HTTPCACHEENABLED = True
HTTPCACHEEXPIRATIONSECS = 86400
HTTPCACHEDIR = "httpcache"


Logging
LOGLEVEL = "INFO"
LOGFILE = "scraper.log"


Auto-throttle for polite scraping
AUTOTHROTTLEENABLED = True
AUTOTHROTTLESTARTDELAY = 1
AUTOTHROTTLEMAXDELAY = 10
AUTOTHROTTLETARGETCONCURRENCY = 2.0

# mldatacollector/spiders/paginatedspider.py
import scrapy

class PaginatedSpider(scrapy.Spider):
    name = "paginated"
    starturls = ["https://example.com/products?page=1"]

    def parse(self, response):
        # Extract items from current page
        for product in response.css("div.product-card"):
            yield {
                "name": product.css("h3.product-name::text").get("").strip(),
                "price": product.css("span.price::text").get("").strip(),
                "description": product.css("p.description::text").get("").strip(),
                "rating": product.css("span.rating::attr(data-value)").get(""),
            }

        # Handle numbered pagination
        currentpage = response.meta.get("page", 1)

        totalpages = int(
            response.css("span.total-pages::text").get("1")
        )

        if currentpage < totalpages:
            nextpage = currentpage + 1
            yield scrapy.Request(
                url=f"https://example.com/products?page={nextpage}",

                callback=self.parse,
                meta={"page": nextpage},
            )

class InfiniteScrollSpider(scrapy.Spider):
    """Handle sites that use API-based infinite scroll."""
    name = "infinitescroll"


    def startrequests(self):
        yield scrapy.Request(
            url="https://example.com/api/products?offset=0&limit=20",
            callback=self.parseapi,

            meta={"offset": 0},
        )

    def parseapi(self, response):
        import json
        data = json.loads(response.text)

        for item in data.get("results", []):
            yield {
                "name": item["name"],
                "price": item["price"],
                "category": item["category"],
            }

        # Continue if more results exist
        offset = response.meta["offset"] + 20
        if data.get("hasmore", False):

            yield scrapy.Request(
                url=f"https://example.com/api/products?offset={offset}&limit=20",
                callback=self.parseapi,
                meta={"offset": offset},
            )

# mldatacollector/spiders/jsspider.py
import scrapy

class JavaScriptSpider(scrapy.Spider):
    """Use Playwright to render JavaScript-heavy pages."""
    name = "jsspider"
    starturls = ["https://example.com/spa-page"]


    customsettings = {
        "DOWNLOADHANDLERS": {

            "http": "scrapyplaywright.handler.ScrapyPlaywrightDownloadHandler",
            "https": "scrapyplaywright.handler.ScrapyPlaywrightDownloadHandler",

        },
        "TWISTEDREACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
        "PLAYWRIGHTBROWSERTYPE": "chromium",
    }

    def startrequests(self):

        for url in self.starturls:
            yield scrapy.Request(
                url,
                meta={
                    "playwright": True,
                    "playwrightpagemethods": [
                        # Wait for content to load
                        {"method": "waitforselector", "args": ["div.content-loaded"]},
                        # Scroll to trigger lazy loading
                        {"method": "evaluate", "args": [
                            "window.scrollTo(0, document.body.scrollHeight)"
                        ]},
                        {"method": "waitfortimeout", "args": [2000]},
                    ],
                },
            )

    def parse(self, response):
        for item in response.css("div.dynamic-content"):
            yield {
                "title": item.css("h2::text").get(""),
                "body": item.css("div.body::text").get(""),
            }

# mldatacollector/utils/cleaning.py
import re
import unicodedata
from typing import Optional

def cleantext(text: str) -> str:

    """Comprehensive text cleaning for ML datasets."""
    if not text:
        return ""

    # Normalize Unicode characters
    text = unicodedata.normalize("NFKD", text)

    # Remove HTML entities
    text = re.sub(r"&[a-zA-Z]+;", " ", text)
    text = re.sub(r"&#\d+;", " ", text)

    # Remove URLs
    text = re.sub(r"https?://\S+", "", text)

    # Remove email addresses
    text = re.sub(r"\S+@\S+\.\S+", "", text)

    # Normalize whitespace
    text = re.sub(r"\s+", " ", text)

    # Remove leading/trailing whitespace
    text = text.strip()

    return text

def cleanprice(pricestr: str) -> Optional[float]:

    """Extract numeric price from string."""
    if not pricestr:
        return None
    cleaned = re.sub(r"[^\d.,]", "", pricestr)

    cleaned = cleaned.replace(",", "")
    try:
        return float(cleaned)
    except ValueError:
        return None

def cleanrating(ratingstr: str) -> Optional[float]:

    """Extract numeric rating from string."""
    if not ratingstr:
        return None
    match = re.search(r"(\d+\.?\d)", ratingstr)

    if match:
        return float(match.group(1))
    return None

def normalizedate(datestr: str) -> Optional[str]:

    """Normalize date to ISO format."""
    from datetime import datetime

    formats = [
        "%Y-%m-%d",
        "%m/%d/%Y",
        "%d/%m/%Y",
        "%B %d, %Y",
        "%b %d, %Y",
        "%Y-%m-%dT%H:%M:%S",
    ]

    for fmt in formats:
        try:
            dt = datetime.strptime(datestr.strip(), fmt)
            return dt.strftime("%Y-%m-%d")
        except ValueError:
            continue

    return None

Storing Data to CSV, JSON, and Database

Built-in Scrapy Export

# Export to JSON
scrapy crawl quotes -o output.json

Export to CSV
scrapy crawl quotes -o output.csv

Export to JSON Lines (better for large datasets)
scrapy crawl quotes -o output.jsonl

Custom Database Pipeline

# mldatacollector/pipelines.py (database pipeline)
from sqlalchemy import createengine, Column, String, Float, Integer, DateTime, Text

from sqlalchemy.orm import sessionmaker, declarativebase
from datetime import datetime

Base = declarativebase()


class ArticleModel(Base):
    tablename = "articles"

    id = Column(Integer, primarykey=True, autoincrement=True)
    title = Column(String(500))
    content = Column(Text)
    author = Column(String(200))
    category = Column(String(100))
    publisheddate = Column(String(50))

    url = Column(String(1000), unique=True)
    wordcount = Column(Integer)
    scrapedat = Column(DateTime, default=datetime.utcnow)


class DatabasePipeline:
    """Store scraped items in a SQL database."""

    def init(self, databaseurl):
        self.databaseurl = databaseurl

    @classmethod
    def fromcrawler(cls, crawler):

        return cls(
            databaseurl=crawler.settings.get(
                "DATABASEURL", "sqlite:///scrapeddata.db"
            )
        )

    def openspider(self, spider):

        self.engine = createengine(self.databaseurl)

        Base.metadata.createall(self.engine)
        Session = sessionmaker(bind=self.engine)
        self.session = Session()

    def closespider(self, spider):

        self.session.commit()
        self.session.close()

    def processitem(self, item, spider):
        from itemadapter import ItemAdapter
        adapter = ItemAdapter(item)

        article = ArticleModel(
            title=adapter.get("title"),
            content=adapter.get("content"),
            author=adapter.get("author"),
            category=adapter.get("category"),
            publisheddate=adapter.get("publisheddate"),
            url=adapter.get("url"),
            wordcount=adapter.get("wordcount"),
        )

        try:
            self.session.add(article)
            self.session.commit()
        except Exception:
            self.session.rollback()

        return item

Ethical Scraping and Rate Limiting

Respecting robots.txt

# Always check robots.txt before scraping
import urllib.robotparser

def canscrape(url: str, useragent: str = "*") -> bool:
    """Check if a URL is allowed by robots.txt."""
    from urllib.parse import urlparse
    parsed = urlparse(url)
    robotsurl = f"{parsed.scheme}://{parsed.netloc}/robots.txt"


    rp = urllib.robotparser.RobotFileParser()
    rp.seturl(robotsurl)

    rp.read()

    return rp.canfetch(useragent, url)


Example usage
url = "https://example.com/products"
if canscrape(url):
    print("Allowed to scrape this URL")
else:
    print("Blocked by robots.txt - do not scrape")

Rate Limiting Implementation

import time
import random
from collections import defaultdict

class RateLimiter:
    """Domain-aware rate limiter for polite scraping."""

    def init(self, mindelay: float = 1.0, maxdelay: float = 3.0):
        self.mindelay = mindelay
        self.maxdelay = maxdelay
        self.lastrequest = defaultdict(float)


    def wait(self, domain: str):
        """Wait an appropriate amount of time before the next request."""
        elapsed = time.time() - self.lastrequest[domain]
        delay = random.uniform(self.mindelay, self.maxdelay)

        if elapsed < delay:
            sleeptime = delay - elapsed

            time.sleep(sleeptime)

        self.lastrequest[domain] = time.time()

# buildsentimentdataset.py
import pandas as pd
from scrapy.crawler import CrawlerProcess
import scrapy

class ReviewSpider(scrapy.Spider):
    name = "reviews"
    starturls = ["https://quotes.toscrape.com/"]
    customsettings = {

        "FEEDS": {"reviewsraw.jsonl": {"format": "jsonlines"}},
        "ROBOTSTXTOBEY": True,

        "DOWNLOADDELAY": 2,
        "AUTOTHROTTLEENABLED": True,

    }

    collecteditems = []

    def parse(self, response):
        for quote in response.css("div.quote"):
            item = {
                "text": quote.css("span.text::text").get(""),
                "author": quote.css("small.author::text").get(""),
                "tags": quote.css("a.tag::text").getall(),
            }
            yield item

        nextpage = response.css("li.next a::attr(href)").get()

        if nextpage:
            yield response.follow(nextpage, callback=self.parse)


Step 1: Run the spider
process = CrawlerProcess()
process.crawl(ReviewSpider)
process.start()

Step 2: Load and clean the data
df = pd.readjson("reviewsraw.jsonl", lines=True)

print(f"Raw data: {len(df)} records")

Remove duplicates
df = df.dropduplicates(subset=["text"])
print(f"After dedup: {len(df)} records")

Remove rows with missing text
df = df.dropna(subset=["text"])

Clean text
df["textclean"] = df["text"].str.strip().str.replace(r"\s+", " ", regex=True)


Step 3: Create train/test split
from sklearn.modelselection import traintestsplit

traindf, testdf = traintestsplit(df, testsize=0.2, randomstate=42)

Step 4: Save ML-ready dataset
traindf.tocsv("datasettrain.csv", index=False)

testdf.tocsv("datasettest.csv", index=False)
print(f"Train set: {len(traindf)} samples")

print(f"Test set: {len(testdf)} samples")

Dataset Validation

def validatedataset(df: pd.DataFrame, textcolumn: str = "textclean") -> dict:
    """Validate a scraped dataset for ML readiness."""
    report = {
        "totalrecords": len(df),
        "nullvalues": df[textcolumn].isnull().sum(),
        "emptystrings": (df[textcolumn] == "").sum(),
        "duplicates": df.duplicated(subset=[textcolumn]).sum(),

        "avgtextlength": df[textcolumn].str.len().mean(),
        "mintextlength": df[textcolumn].str.len().min(),

        "maxtextlength": df[textcolumn].str.len().max(),
    }

    print("Dataset Validation Report:")
    for key, value in report.items():
        print(f"  {key}: {value}")

    # Quality checks
    if report["nullvalues"] > 0:

        print("  [WARNING] Dataset contains null values")
    if report["duplicates"] > 0:
        print("  [WARNING] Dataset contains duplicates")
    if report["mintextlength"] < 10:

        print("  [WARNING] Some texts are very short")

    return report

validatedataset(traindf)

Conclusion

Scrapy and BeautifulSoup form a powerful combination for collecting web data for machine learning projects. BeautifulSoup excels at quick parsing tasks and prototyping, while Scrapy provides the infrastructure needed for large-scale, production-grade crawling with built-in support for rate limiting, pipelines, and middleware.

The key to successful web scraping for ML is treating it as a data engineering problem: define your schema upfront, build robust cleaning pipelines, validate your output, and always scrape ethically. With the techniques covered in this tutorial, you have everything you need to build high-quality ML datasets from web data.

Start with BeautifulSoup for small-scale exploration, graduate to Scrapy when you need scale and reliability, and always prioritize data quality over quantity.