pip install firecrawl-py

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEYHERE")

export FIRECRAWLAPIKEY="fc-YOURAPIKEYHERE"

import os
from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey=os.getenv("FIRECRAWLAPIKEY"))

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Scrape a single page
result = app.scrapeurl(
    "https://docs.python.org/3/tutorial/classes.html",
    params={"formats": ["markdown"]}
)

Access markdown content
markdowncontent = result.get("markdown", "")

print(f"Content length: {len(markdowncontent)} characters")
print(markdowncontent[:500])


Page metadata
metadata = result.get("metadata", {})
print(f"Title: {metadata.get('title')}")
print(f"Description: {metadata.get('description')}")
print(f"Language: {metadata.get('language')}")

result = app.scrapeurl(
    "https://example.com/article",
    params={"formats": ["html", "markdown"]}
)

htmlcontent = result.get("html", "")

markdowncontent = result.get("markdown", "")

print(f"HTML length: {len(htmlcontent)}")

print(f"Markdown length: {len(markdowncontent)}")

result = app.scrapeurl(
    "https://example.com",
    params={"formats": ["screenshot"]}
)

screenshoturl = result.get("screenshot")
print(f"Screenshot URL: {screenshoturl}")


Download screenshot
import requests
if screenshoturl:
    response = requests.get(screenshoturl)

    with open("screenshot.png", "wb") as f:
        f.write(response.content)
    print("Screenshot saved!")

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

Crawl website with configuration
crawlresult = app.crawlurl(
    "https://docs.python.org/3/tutorial/",
    params={
        "limit": 50,              # Maximum 50 pages
        "maxDepth": 3,            # Maximum link depth
        "includePaths": ["/3/tutorial/"],  # Only specific paths
        "excludePaths": ["/3/tutorial/index.html"],
        "scrapeOptions": {
            "formats": ["markdown"]
        }
    },
    pollinterval=5  # Check status every 5 seconds

)

Process results
pages = crawlresult.get("data", [])
print(f"Total pages crawled: {len(pages)}")

for page in pages:
    metadata = page.get("metadata", {})
    markdown = page.get("markdown", "")
    url = metadata.get("sourceURL", "unknown")
    print(f"  - {url}: {len(markdown)} chars")

from firecrawl import FirecrawlApp
import time

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Start async crawl
crawlstatus = app.asynccrawlurl(
    "https://docs.langchain.com/",
    params={
        "limit": 100,
        "maxDepth": 3,
        "scrapeOptions": {
            "formats": ["markdown"]
        }
    }
)

crawlid = crawlstatus.get("id")
print(f"Crawl started with ID: {crawlid}")


Poll for status
while True:
    status = app.checkcrawlstatus(crawlid)
    state = status.get("status")
    completed = status.get("completed", 0)
    total = status.get("total", 0)

    print(f"Status: {state} - {completed}/{total} pages")

    if state == "completed":
        pages = status.get("data", [])
        print(f"Crawl complete! Total: {len(pages)} pages")
        break

    time.sleep(10)

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Map all URLs
mapresult = app.mapurl(

    "https://docs.python.org/3/",
    params={"limit": 500}
)

urls = mapresult.get("links", [])
print(f"Total URLs found: {len(urls)}")

Filter relevant URLs
tutorialurls = [url for url in urls if "/tutorial/" in url]

print(f"Tutorial URLs: {len(tutorialurls)}")

for url in tutorialurls[:10]:

    print(f"  - {url}")

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

Define extraction schema
schema = {
    "type": "object",
    "properties": {
        "title": {"type": "string", "description": "Article title"},
        "author": {"type": "string", "description": "Author name"},
        "publisheddate": {"type": "string", "description": "Publication date"},

        "summary": {"type": "string", "description": "Article summary in 2-3 sentences"},
        "topics": {
            "type": "array",
            "items": {"type": "string"},
            "description": "Main topics discussed"
        },
        "codelanguages": {
            "type": "array",
            "items": {"type": "string"},
            "description": "Programming languages used in code examples"
        }
    },
    "required": ["title", "summary", "topics"]
}

result = app.scrapeurl(

    "https://blog.example.com/machine-learning-tutorial",
    params={
        "formats": ["extract"],
        "extract": {
            "schema": schema,
            "prompt": "Extract the key information from this technical blog post."
        }
    }
)

extracted = result.get("extract", {})
print(f"Title: {extracted.get('title')}")
print(f"Author: {extracted.get('author')}")
print(f"Summary: {extracted.get('summary')}")
print(f"Topics: {', '.join(extracted.get('topics', []))}")

productschema = {
    "type": "object",
    "properties": {
        "productname": {"type": "string"},

        "price": {"type": "number"},
        "currency": {"type": "string"},
        "rating": {"type": "number"},
        "reviewcount": {"type": "integer"},
        "features": {
            "type": "array",
            "items": {"type": "string"}
        },
        "availability": {"type": "string"}
    }
}

result = app.scrapeurl(

    "https://shop.example.com/product/gpu-rtx4090",
    params={
        "formats": ["extract"],
        "extract": {
            "schema": productschema,
            "prompt": "Extract product information including price, rating, and features."
        }
    }
)

product = result.get("extract", {})
print(f"Product: {product.get('productname')}")

print(f"Price: {product.get('currency', '
    

Batch Scraping

For scraping many URLs at once:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


urls = [
    "https://docs.python.org/3/tutorial/classes.html",
    "https://docs.python.org/3/tutorial/modules.html",
    "https://docs.python.org/3/tutorial/errors.html",
    "https://docs.python.org/3/tutorial/inputoutput.html",
    "https://docs.python.org/3/tutorial/stdlib.html",
]

Batch scrape
batchresult = app.batchscrapeurls(
    urls,
    params={"formats": ["markdown"]}
)

Process results
pages = batchresult.get("data", [])

for page in pages:
    metadata = page.get("metadata", {})
    title = metadata.get("title", "Unknown")
    contentlength = len(page.get("markdown", ""))
    print(f"  {title}: {contentlength} chars")

# Start batch asynchronously
batchstatus = app.asyncbatchscrapeurls(

    urls,
    params={"formats": ["markdown"]}
)

batchid = batchstatus.get("id")

print(f"Batch started with ID: {batchid}")

Check status
import time
while True:
    status = app.checkbatchscrapestatus(batchid)
    state = status.get("status")
    completed = status.get("completed", 0)
    total = status.get("total", 0)

    print(f"Progress: {completed}/{total}")

    if state == "completed":
        results = status.get("data", [])
        print(f"Batch complete! {len(results)} pages scraped successfully")
        break

    time.sleep(5)

Webhook Notifications

For long-running crawl operations, use webhooks to get notified when complete:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Crawl with webhook
crawlstatus = app.asynccrawlurl(
    "https://docs.example.com/",
    params={
        "limit": 200,
        "maxDepth": 4,
        "scrapeOptions": {
            "formats": ["markdown"]
        },
        "webhook": "https://your-server.com/webhook/firecrawl"
    }
)

print(f"Crawl started: {crawlstatus.get('id')}")

print("Webhook will be called when complete")

from fastapi import FastAPI, Request

app = FastAPI()

@app.post("/webhook/firecrawl")
async def firecrawlwebhook(request: Request):
    data = await request.json()

    eventtype = data.get("type")

    crawlid = data.get("id")

    if eventtype == "crawl.completed":

        pages = data.get("data", [])
        print(f"Crawl {crawlid} completed: {len(pages)} pages")

        # Process pages
        for page in pages:
            # Save to database or vector store
            pass

    elif eventtype == "crawl.failed":

        error = data.get("error")
        print(f"Crawl {crawlid} failed: {error}")

    return {"status": "received"}

Handling JavaScript-Rendered Pages

Firecrawl handles JavaScript rendering by default. However, for pages that require longer loading times:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Wait for specific elements before scraping
result = app.scrapeurl(
    "https://app.example.com/dashboard",
    params={
        "formats": ["markdown"],
        "waitFor": 5000,  # Wait 5 seconds
    }
)

print(result.get("markdown", "")[:500])

Handling Pages with Infinite Scroll

result = app.scrapeurl(
    "https://feed.example.com",
    params={
        "formats": ["markdown"],
        "waitFor": 3000,
        "actions": [
            {"type": "scroll", "direction": "down", "amount": 3},
            {"type": "wait", "milliseconds": 2000},
            {"type": "scroll", "direction": "down", "amount": 3},
            {"type": "wait", "milliseconds": 2000}
        ]
    }
)

import time
from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

def scrapewithratelimit(urls, delay=2.0):

    """Scrape multiple URLs with rate limiting."""
    results = []

    for i, url in enumerate(urls):
        try:
            result = app.scrapeurl(url, params={"formats": ["markdown"]})
            results.append({
                "url": url,
                "success": True,
                "content": result.get("markdown", ""),
                "metadata": result.get("metadata", {})
            })
            print(f"[{i+1}/{len(urls)}] OK: {url}")
        except Exception as e:
            results.append({
                "url": url,
                "success": False,
                "error": str(e)
            })
            print(f"[{i+1}/{len(urls)}] ERROR: {url} - {e}")

        if i < len(urls) - 1:
            time.sleep(delay)

    return results

Local Caching

import json
import hashlib
import os
from datetime import datetime, timedelta
from firecrawl import FirecrawlApp

class CachedFirecrawl:
    """Firecrawl wrapper with local caching."""

    def init(self, apikey, cachedir="./firecrawlcache", ttlhours=24):
        self.app = FirecrawlApp(apikey=apikey)
        self.cachedir = cachedir
        self.ttl = timedelta(hours=ttlhours)

        os.makedirs(cachedir, existok=True)


    def cachekey(self, url, params):

        """Generate cache key from URL and params."""
        keystring = f"{url}:{json.dumps(params, sortkeys=True)}"

        return hashlib.md5(keystring.encode()).hexdigest()

    def getcache(self, cachekey):

        """Retrieve data from cache if still valid."""
        cachefile = os.path.join(self.cachedir, f"{cachekey}.json")
        if os.path.exists(cachefile):

            with open(cachefile, 'r') as f:
                cached = json.load(f)

            cachedtime = datetime.fromisoformat(cached["timestamp"])

            if datetime.now() - cachedtime < self.ttl:
                return cached["data"]
        return None

    def setcache(self, cachekey, data):

        """Save data to cache."""
        cachefile = os.path.join(self.cachedir, f"{cachekey}.json")
        with open(cachefile, 'w') as f:

            json.dump({
                "timestamp": datetime.now().isoformat(),
                "data": data
            }, f)

    def scrapeurl(self, url, params=None):
        """Scrape with caching."""
        params = params or {"formats": ["markdown"]}
        cachekey = self.cachekey(url, params)


        cached = self.getcache(cachekey)
        if cached:
            print(f"Cache hit: {url}")
            return cached

        print(f"Cache miss, scraping: {url}")
        result = self.app.scrapeurl(url, params=params)

        self.setcache(cachekey, result)
        return result

Usage
scraper = CachedFirecrawl(apikey="fc-YOURAPIKEY", ttlhours=48)

First time: scrape from Firecrawl
result1 = scraper.scrapeurl("https://docs.python.org/3/tutorial/classes.html")


Second time: from cache
result2 = scraper.scrapeurl("https://docs.python.org/3/tutorial/classes.html")

Self-Hosted Option

Firecrawl can also be deployed self-hosted:

# Clone repository
git clone https://github.com/mendableai/firecrawl.git
cd firecrawl

Setup environment
cp .env.example .env
Edit .env as needed

Run with Docker Compose
docker-compose up -d

# Connect to self-hosted Firecrawl
from firecrawl import FirecrawlApp

app = FirecrawlApp(
    apikey="your-self-hosted-key",

    apiurl="http://localhost:3002"
)

result = app.scrapeurl("https://example.com", params={"formats": ["markdown"]})

print(result.get("markdown", "")[:200])

import os
import json
import hashlib
from datetime import datetime
from firecrawl import FirecrawlApp

class KnowledgeBaseBuilder:
    """Pipeline for building a knowledge base from websites."""

    def init(self, firecrawlapikey):

        self.firecrawl = FirecrawlApp(apikey=firecrawlapikey)
        self.documents = []

    def discoverurls(self, baseurl, limit=200):
        """Discover all URLs within a website."""
        print(f"Mapping URLs from {baseurl}...")


        mapresult = self.firecrawl.mapurl(

            baseurl,
            params={"limit": limit}
        )

        urls = mapresult.get("links", [])

        print(f"Found {len(urls)} URLs")
        return urls

    def crawlwebsite(self, baseurl, limit=100, maxdepth=3):
        """Crawl website and extract content."""
        print(f"Starting crawl of {baseurl}...")


        result = self.firecrawl.crawlurl(
            baseurl,

            params={
                "limit": limit,
                "maxDepth": maxdepth,
                "scrapeOptions": {
                    "formats": ["markdown"]
                }
            },
            pollinterval=10

        )

        pages = result.get("data", [])
        print(f"Successfully crawled {len(pages)} pages")

        for page in pages:
            metadata = page.get("metadata", {})
            markdown = page.get("markdown", "")

            if markdown.strip():
                doc = {
                    "url": metadata.get("sourceURL", ""),
                    "title": metadata.get("title", ""),
                    "description": metadata.get("description", ""),
                    "content": markdown,
                    "contentlength": len(markdown),
                    "crawledat": datetime.now().isoformat(),

                    "docid": hashlib.md5(
                        metadata.get("sourceURL", "").encode()
                    ).hexdigest()
                }
                self.documents.append(doc)

        return self.documents

    def extractstructureddata(self, urls, schema):
        """Extract structured data from specific pages."""
        extracted = []

        for i, url in enumerate(urls):
            try:
                result = self.firecrawl.scrapeurl(

                    url,
                    params={
                        "formats": ["extract"],
                        "extract": {
                            "schema": schema,
                            "prompt": "Extract the structured information accurately."
                        }
                    }
                )

                data = result.get("extract", {})
                data["sourceurl"] = url
                extracted.append(data)
                print(f"  [{i+1}/{len(urls)}] Extracted: {url}")

            except Exception as e:
                print(f"  [{i+1}/{len(urls)}] Error: {url} - {e}")

        return extracted

    def chunkdocuments(self, chunksize=1000, overlap=200):
        """Split documents into chunks for vector store."""
        chunks = []

        for doc in self.documents:
            content = doc["content"]

            # Split by paragraphs first
            paragraphs = content.split("\n\n")
            currentchunk = ""


            for para in paragraphs:
                if len(currentchunk) + len(para) > chunksize and currentchunk:
                    chunks.append({
                        "docid": doc["docid"],
                        "url": doc["url"],
                        "title": doc["title"],
                        "chunktext": currentchunk.strip(),
                        "chunkid": hashlib.md5(

                            currentchunk.encode()
                        ).hexdigest()
                    })

                    # Overlap: take the end of the previous chunk
                    words = currentchunk.split()

                    overlapwords = words[-overlap // 5:] if len(words) > overlap // 5 else words
                    currentchunk = " ".join(overlapwords) + "\n\n" + para
                else:
                    currentchunk += "\n\n" + para if currentchunk else para

            # Last chunk
            if currentchunk.strip():

                chunks.append({
                    "docid": doc["docid"],

                    "url": doc["url"],
                    "title": doc["title"],
                    "chunktext": currentchunk.strip(),

                    "chunkid": hashlib.md5(
                        currentchunk.encode()

                    ).hexdigest()
                })

        print(f"Total chunks: {len(chunks)} from {len(self.documents)} documents")
        return chunks

    def storetochromadb(self, chunks):

        """Store chunks in ChromaDB vector store."""
        import chromadb
        from chromadb.utils import embeddingfunctions

        # Initialize ChromaDB
        client = chromadb.PersistentClient(path="./knowledgebasedb")

        # Use sentence-transformers for embedding
        embeddingfn = embeddingfunctions.SentenceTransformerEmbeddingFunction(
            modelname="all-MiniLM-L6-v2"

        )

        collection = client.getorcreatecollection(
            name="knowledgebase",

            embeddingfunction=embeddingfn,

            metadata={"description": "Knowledge base from documentation website"}
        )

        # Batch insert
        batchsize = 50
        for i in range(0, len(chunks), batchsize):

            batch = chunks[i:i + batchsize]

            collection.add(
                ids=[c["chunkid"] for c in batch],

                documents=[c["chunktext"] for c in batch],
                metadatas=[{
                    "url": c["url"],
                    "title": c["title"],
                    "docid": c["docid"]
                } for c in batch]
            )

            print(f"  Inserted batch {i // batchsize + 1}: "

                  f"{len(batch)} chunks")

        print(f"Total {len(chunks)} chunks stored in ChromaDB")
        return collection

    def queryknowledgebase(self, collection, query, nresults=5):
        """Query the knowledge base."""
        results = collection.query(
            querytexts=[query],

            nresults=nresults

        )

        print(f"Query: '{query}'")
        print(f"Results ({len(results['documents'][0])}):\n")

        for i, (doc, meta, dist) in enumerate(zip(
            results["documents"][0],
            results["metadatas"][0],
            results["distances"][0]
        )):
            print(f"  {i+1}. [{meta['title']}]({meta['url']})")
            print(f"     Distance: {dist:.4f}")
            print(f"     Preview: {doc[:150]}...")
            print()

        return results

    def savedocuments(self, outputpath):

        """Save documents to JSON file."""
        with open(outputpath, 'w', encoding='utf-8') as f:
            json.dump(self.documents, f, ensureascii=False, indent=2)

        print(f"Documents saved to {outputpath}")

Complete pipeline
def runknowledgebasepipeline():

    """Run the complete knowledge base pipeline."""

    builder = KnowledgeBaseBuilder(
        firecrawlapikey="fc-YOURAPIKEY"

    )

    # Step 1: Crawl documentation website
    documents = builder.crawlwebsite(
        "https://docs.example.com/",
        limit=50,
        maxdepth=3

    )

    # Step 2: Save raw documents
    builder.savedocuments("./knowledgebaseraw.json")

    # Step 3: Chunk documents
    chunks = builder.chunkdocuments(chunksize=1000, overlap=200)

    # Step 4: Store in vector store
    collection = builder.storetochromadb(chunks)

    # Step 5: Test query
    builder.queryknowledgebase(
        collection,
        "How to configure authentication?"
    )

    print("\nKnowledge base built successfully!")

Run
runknowledgebasepipeline()

from openai import OpenAI
import chromadb
from chromadb.utils import embeddingfunctions

def ragquery(question, collection, openaiclient, ncontext=5):

    """RAG query using the knowledge base."""

    # Retrieve context from knowledge base
    results = collection.query(
        querytexts=[question],
        nresults=ncontext
    )

    # Combine context
    contextparts = []

    for doc, meta in zip(results["documents"][0], results["metadatas"][0]):
        contextparts.append(f"Source: {meta['title']} ({meta['url']})\n{doc}")

    context = "\n\n".join(contextparts)


    # Generate answer with LLM
    response = openaiclient.chat.completions.create(
        model="gpt-4",
        messages=[
            {
                "role": "system",
                "content": (
                    "You are an assistant that answers questions based on the "
                    "provided context. Answer accurately based on the context. "
                    "If the information is not in the context, say so honestly."
                )
            },
            {
                "role": "user",
                "content": f"Context:\n{context}\n\nQuestion: {question}"
            }
        ],
        temperature=0.1
    )

    answer = response.choices[0].message.content

    # Return answer with sources
    sources = [
        {"title": m["title"], "url": m["url"]}
        for m in results["metadatas"][0]
    ]

    return {
        "answer": answer,
        "sources": sources
    }

Usage
openaiclient = OpenAI(apikey="sk-...")
result = ragquery(

"How do I set up authentication in the app?",
collection,
openaiclient
)
print(result["answer"])
print("Sources:", result["sources"])

Tips and Best Practices

Conclusion

Firecrawl simplifies the web scraping process for AI and LLM needs. With clean markdown output, JavaScript rendering capabilities, and structured data extraction features, Firecrawl becomes an invaluable tool for building AI data pipelines.

Combining Firecrawl with vector databases like ChromaDB creates a strong foundation for RAG systems. Webhook and batch processing features make it suitable for production pipelines that require reliability and scalability.

Choose between the managed API for convenience or self-hosted for full control, depending on your project needs and your organization's data policies.