pip install firecrawl-py

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEYHERE")

export FIRECRAWLAPIKEY="fc-YOURAPIKEYHERE"

import os
from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey=os.getenv("FIRECRAWLAPIKEY"))

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Scrape halaman tunggal
result = app.scrapeurl(
    "https://docs.python.org/3/tutorial/classes.html",
    params={"formats": ["markdown"]}
)

Akses konten markdown
markdowncontent = result.get("markdown", "")

print(f"Panjang konten: {len(markdowncontent)} karakter")
print(markdowncontent[:500])


Metadata halaman
metadata = result.get("metadata", {})
print(f"Title: {metadata.get('title')}")
print(f"Description: {metadata.get('description')}")
print(f"Language: {metadata.get('language')}")

result = app.scrapeurl(
    "https://example.com/article",
    params={"formats": ["html", "markdown"]}
)

htmlcontent = result.get("html", "")

markdowncontent = result.get("markdown", "")

print(f"HTML length: {len(htmlcontent)}")

print(f"Markdown length: {len(markdowncontent)}")

result = app.scrapeurl(
    "https://example.com",
    params={"formats": ["screenshot"]}
)

screenshoturl = result.get("screenshot")
print(f"Screenshot URL: {screenshoturl}")


Download screenshot
import requests
if screenshoturl:
    response = requests.get(screenshoturl)

    with open("screenshot.png", "wb") as f:
        f.write(response.content)
    print("Screenshot saved!")

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

Crawl website dengan konfigurasi
crawlresult = app.crawlurl(
    "https://docs.python.org/3/tutorial/",
    params={
        "limit": 50,              # Maksimal 50 halaman
        "maxDepth": 3,            # Kedalaman link maksimal
        "includePaths": ["/3/tutorial/"],  # Hanya path tertentu
        "excludePaths": ["/3/tutorial/index.html"],
        "scrapeOptions": {
            "formats": ["markdown"]
        }
    },
    pollinterval=5  # Cek status setiap 5 detik

)

Proses hasil
pages = crawlresult.get("data", [])
print(f"Total halaman di-crawl: {len(pages)}")

for page in pages:
    metadata = page.get("metadata", {})
    markdown = page.get("markdown", "")
    url = metadata.get("sourceURL", "unknown")
    print(f"  - {url}: {len(markdown)} chars")

from firecrawl import FirecrawlApp
import time

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Mulai crawl asinkron
crawlstatus = app.asynccrawlurl(
    "https://docs.langchain.com/",
    params={
        "limit": 100,
        "maxDepth": 3,
        "scrapeOptions": {
            "formats": ["markdown"]
        }
    }
)

crawlid = crawlstatus.get("id")
print(f"Crawl dimulai dengan ID: {crawlid}")


Polling status
while True:
    status = app.checkcrawlstatus(crawlid)
    state = status.get("status")
    completed = status.get("completed", 0)
    total = status.get("total", 0)

    print(f"Status: {state} - {completed}/{total} halaman")

    if state == "completed":
        pages = status.get("data", [])
        print(f"Crawl selesai! Total: {len(pages)} halaman")
        break

    time.sleep(10)

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Map semua URL
mapresult = app.mapurl(

    "https://docs.python.org/3/",
    params={"limit": 500}
)

urls = mapresult.get("links", [])
print(f"Total URL ditemukan: {len(urls)}")

Filter URL yang relevan
tutorialurls = [url for url in urls if "/tutorial/" in url]

print(f"URL tutorial: {len(tutorialurls)}")

for url in tutorialurls[:10]:

    print(f"  - {url}")

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

Definisikan schema untuk ekstraksi
schema = {
    "type": "object",
    "properties": {
        "title": {"type": "string", "description": "Judul artikel"},
        "author": {"type": "string", "description": "Nama penulis"},
        "publisheddate": {"type": "string", "description": "Tanggal publikasi"},

        "summary": {"type": "string", "description": "Ringkasan artikel dalam 2-3 kalimat"},
        "topics": {
            "type": "array",
            "items": {"type": "string"},
            "description": "Topik utama yang dibahas"
        },
        "codelanguages": {
            "type": "array",
            "items": {"type": "string"},
            "description": "Bahasa pemrograman yang digunakan dalam contoh kode"
        }
    },
    "required": ["title", "summary", "topics"]
}

result = app.scrapeurl(

    "https://blog.example.com/machine-learning-tutorial",
    params={
        "formats": ["extract"],
        "extract": {
            "schema": schema,
            "prompt": "Extract the key information from this technical blog post."
        }
    }
)

extracted = result.get("extract", {})
print(f"Title: {extracted.get('title')}")
print(f"Author: {extracted.get('author')}")
print(f"Summary: {extracted.get('summary')}")
print(f"Topics: {', '.join(extracted.get('topics', []))}")

productschema = {
    "type": "object",
    "properties": {
        "productname": {"type": "string"},

        "price": {"type": "number"},
        "currency": {"type": "string"},
        "rating": {"type": "number"},
        "reviewcount": {"type": "integer"},
        "features": {
            "type": "array",
            "items": {"type": "string"}
        },
        "availability": {"type": "string"}
    }
}

result = app.scrapeurl(

    "https://shop.example.com/product/gpu-rtx4090",
    params={
        "formats": ["extract"],
        "extract": {
            "schema": productschema,
            "prompt": "Extract product information including price, rating, and features."
        }
    }
)

product = result.get("extract", {})
print(f"Product: {product.get('productname')}")

print(f"Price: {product.get('currency', '
    

Batch Scraping

Untuk scraping banyak URL sekaligus:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


urls = [
    "https://docs.python.org/3/tutorial/classes.html",
    "https://docs.python.org/3/tutorial/modules.html",
    "https://docs.python.org/3/tutorial/errors.html",
    "https://docs.python.org/3/tutorial/inputoutput.html",
    "https://docs.python.org/3/tutorial/stdlib.html",
]

Batch scrape
batchresult = app.batchscrapeurls(
    urls,
    params={"formats": ["markdown"]}
)

Proses hasil
pages = batchresult.get("data", [])

for page in pages:
    metadata = page.get("metadata", {})
    title = metadata.get("title", "Unknown")
    contentlength = len(page.get("markdown", ""))
    print(f"  {title}: {contentlength} chars")

# Mulai batch secara asinkron
batchstatus = app.asyncbatchscrapeurls(

    urls,
    params={"formats": ["markdown"]}
)

batchid = batchstatus.get("id")

print(f"Batch dimulai dengan ID: {batchid}")

Cek status
import time
while True:
    status = app.checkbatchscrapestatus(batchid)
    state = status.get("status")
    completed = status.get("completed", 0)
    total = status.get("total", 0)

    print(f"Progress: {completed}/{total}")

    if state == "completed":
        results = status.get("data", [])
        print(f"Batch selesai! {len(results)} halaman berhasil di-scrape")
        break

    time.sleep(5)

Webhook Notifications

Untuk operasi crawl yang lama, gunakan webhook agar mendapat notifikasi saat selesai:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Crawl dengan webhook
crawlstatus = app.asynccrawlurl(
    "https://docs.example.com/",
    params={
        "limit": 200,
        "maxDepth": 4,
        "scrapeOptions": {
            "formats": ["markdown"]
        },
        "webhook": "https://your-server.com/webhook/firecrawl"
    }
)

print(f"Crawl dimulai: {crawlstatus.get('id')}")

print("Webhook akan dipanggil saat selesai")

from fastapi import FastAPI, Request

app = FastAPI()

@app.post("/webhook/firecrawl")
async def firecrawlwebhook(request: Request):
    data = await request.json()

    eventtype = data.get("type")

    crawlid = data.get("id")

    if eventtype == "crawl.completed":

        pages = data.get("data", [])
        print(f"Crawl {crawlid} selesai: {len(pages)} halaman")

        # Proses halaman
        for page in pages:
            # Simpan ke database atau vector store
            pass

    elif eventtype == "crawl.failed":

        error = data.get("error")
        print(f"Crawl {crawlid} gagal: {error}")

    return {"status": "received"}

Handling JavaScript-Rendered Pages

Firecrawl secara default sudah menangani JavaScript rendering. Namun, untuk halaman yang memerlukan waktu loading lebih lama:

from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")


Tunggu elemen spesifik sebelum scraping
result = app.scrapeurl(
    "https://app.example.com/dashboard",
    params={
        "formats": ["markdown"],
        "waitFor": 5000,  # Tunggu 5 detik
    }
)

print(result.get("markdown", "")[:500])

Menangani Halaman dengan Infinite Scroll

result = app.scrapeurl(
    "https://feed.example.com",
    params={
        "formats": ["markdown"],
        "waitFor": 3000,
        "actions": [
            {"type": "scroll", "direction": "down", "amount": 3},
            {"type": "wait", "milliseconds": 2000},
            {"type": "scroll", "direction": "down", "amount": 3},
            {"type": "wait", "milliseconds": 2000}
        ]
    }
)

import time
from firecrawl import FirecrawlApp

app = FirecrawlApp(apikey="fc-YOURAPIKEY")

def scrapewithratelimit(urls, delay=2.0):

    """Scrape multiple URLs dengan rate limiting."""
    results = []

    for i, url in enumerate(urls):
        try:
            result = app.scrapeurl(url, params={"formats": ["markdown"]})
            results.append({
                "url": url,
                "success": True,
                "content": result.get("markdown", ""),
                "metadata": result.get("metadata", {})
            })
            print(f"[{i+1}/{len(urls)}] OK: {url}")
        except Exception as e:
            results.append({
                "url": url,
                "success": False,
                "error": str(e)
            })
            print(f"[{i+1}/{len(urls)}] ERROR: {url} - {e}")

        if i < len(urls) - 1:
            time.sleep(delay)

    return results

Caching Lokal

import json
import hashlib
import os
from datetime import datetime, timedelta
from firecrawl import FirecrawlApp

class CachedFirecrawl:
    """Firecrawl wrapper dengan caching lokal."""

    def init(self, apikey, cachedir="./firecrawlcache", ttlhours=24):
        self.app = FirecrawlApp(apikey=apikey)
        self.cachedir = cachedir
        self.ttl = timedelta(hours=ttlhours)

        os.makedirs(cachedir, existok=True)


    def cachekey(self, url, params):

        """Generate cache key dari URL dan params."""
        keystring = f"{url}:{json.dumps(params, sortkeys=True)}"

        return hashlib.md5(keystring.encode()).hexdigest()

    def getcache(self, cachekey):

        """Ambil data dari cache jika masih valid."""
        cachefile = os.path.join(self.cachedir, f"{cachekey}.json")
        if os.path.exists(cachefile):

            with open(cachefile, 'r') as f:
                cached = json.load(f)

            cachedtime = datetime.fromisoformat(cached["timestamp"])

            if datetime.now() - cachedtime < self.ttl:
                return cached["data"]
        return None

    def setcache(self, cachekey, data):

        """Simpan data ke cache."""
        cachefile = os.path.join(self.cachedir, f"{cachekey}.json")
        with open(cachefile, 'w') as f:

            json.dump({
                "timestamp": datetime.now().isoformat(),
                "data": data
            }, f)

    def scrapeurl(self, url, params=None):
        """Scrape dengan caching."""
        params = params or {"formats": ["markdown"]}
        cachekey = self.cachekey(url, params)


        cached = self.getcache(cachekey)
        if cached:
            print(f"Cache hit: {url}")
            return cached

        print(f"Cache miss, scraping: {url}")
        result = self.app.scrapeurl(url, params=params)

        self.setcache(cachekey, result)
        return result

Penggunaan
scraper = CachedFirecrawl(apikey="fc-YOURAPIKEY", ttlhours=48)

Pertama kali: scrape dari Firecrawl
result1 = scraper.scrapeurl("https://docs.python.org/3/tutorial/classes.html")


Kedua kali: dari cache
result2 = scraper.scrapeurl("https://docs.python.org/3/tutorial/classes.html")

Self-Hosted Option

Firecrawl juga bisa di-deploy secara self-hosted:

# Clone repository
git clone https://github.com/mendableai/firecrawl.git
cd firecrawl

Setup environment
cp .env.example .env
Edit .env sesuai kebutuhan

Jalankan dengan Docker Compose
docker-compose up -d

# Koneksi ke self-hosted Firecrawl
from firecrawl import FirecrawlApp

app = FirecrawlApp(
    apikey="your-self-hosted-key",

    apiurl="http://localhost:3002"
)

result = app.scrapeurl("https://example.com", params={"formats": ["markdown"]})

print(result.get("markdown", "")[:200])

import os
import json
import hashlib
from datetime import datetime
from firecrawl import FirecrawlApp

class KnowledgeBaseBuilder:
    """Pipeline untuk membangun knowledge base dari website."""

    def init(self, firecrawlapikey):

        self.firecrawl = FirecrawlApp(apikey=firecrawlapikey)
        self.documents = []

    def discoverurls(self, baseurl, limit=200):
        """Temukan semua URL dalam website."""
        print(f"Mapping URLs dari {baseurl}...")


        mapresult = self.firecrawl.mapurl(

            baseurl,
            params={"limit": limit}
        )

        urls = mapresult.get("links", [])

        print(f"Ditemukan {len(urls)} URLs")
        return urls

    def crawlwebsite(self, baseurl, limit=100, maxdepth=3):
        """Crawl website dan ekstrak konten."""
        print(f"Memulai crawl {baseurl}...")


        result = self.firecrawl.crawlurl(
            baseurl,

            params={
                "limit": limit,
                "maxDepth": maxdepth,
                "scrapeOptions": {
                    "formats": ["markdown"]
                }
            },
            pollinterval=10

        )

        pages = result.get("data", [])
        print(f"Berhasil crawl {len(pages)} halaman")

        for page in pages:
            metadata = page.get("metadata", {})
            markdown = page.get("markdown", "")

            if markdown.strip():
                doc = {
                    "url": metadata.get("sourceURL", ""),
                    "title": metadata.get("title", ""),
                    "description": metadata.get("description", ""),
                    "content": markdown,
                    "contentlength": len(markdown),
                    "crawledat": datetime.now().isoformat(),

                    "docid": hashlib.md5(
                        metadata.get("sourceURL", "").encode()
                    ).hexdigest()
                }
                self.documents.append(doc)

        return self.documents

    def extractstructureddata(self, urls, schema):
        """Ekstrak data terstruktur dari halaman tertentu."""
        extracted = []

        for i, url in enumerate(urls):
            try:
                result = self.firecrawl.scrapeurl(

                    url,
                    params={
                        "formats": ["extract"],
                        "extract": {
                            "schema": schema,
                            "prompt": "Extract the structured information accurately."
                        }
                    }
                )

                data = result.get("extract", {})
                data["sourceurl"] = url
                extracted.append(data)
                print(f"  [{i+1}/{len(urls)}] Extracted: {url}")

            except Exception as e:
                print(f"  [{i+1}/{len(urls)}] Error: {url} - {e}")

        return extracted

    def chunkdocuments(self, chunksize=1000, overlap=200):
        """Pecah dokumen menjadi chunks untuk vector store."""
        chunks = []

        for doc in self.documents:
            content = doc["content"]

            # Split berdasarkan paragraf terlebih dahulu
            paragraphs = content.split("\n\n")
            currentchunk = ""


            for para in paragraphs:
                if len(currentchunk) + len(para) > chunksize and currentchunk:
                    chunks.append({
                        "docid": doc["docid"],
                        "url": doc["url"],
                        "title": doc["title"],
                        "chunktext": currentchunk.strip(),
                        "chunkid": hashlib.md5(

                            currentchunk.encode()
                        ).hexdigest()
                    })

                    # Overlap: ambil bagian akhir chunk sebelumnya
                    words = currentchunk.split()

                    overlapwords = words[-overlap // 5:] if len(words) > overlap // 5 else words
                    currentchunk = " ".join(overlapwords) + "\n\n" + para
                else:
                    currentchunk += "\n\n" + para if currentchunk else para

            # Chunk terakhir
            if currentchunk.strip():

                chunks.append({
                    "docid": doc["docid"],

                    "url": doc["url"],
                    "title": doc["title"],
                    "chunktext": currentchunk.strip(),

                    "chunkid": hashlib.md5(
                        currentchunk.encode()

                    ).hexdigest()
                })

        print(f"Total chunks: {len(chunks)} dari {len(self.documents)} dokumen")
        return chunks

    def storetochromadb(self, chunks):

        """Simpan chunks ke ChromaDB vector store."""
        import chromadb
        from chromadb.utils import embeddingfunctions

        # Inisialisasi ChromaDB
        client = chromadb.PersistentClient(path="./knowledgebasedb")

        # Gunakan sentence-transformers untuk embedding
        embeddingfn = embeddingfunctions.SentenceTransformerEmbeddingFunction(
            modelname="all-MiniLM-L6-v2"

        )

        collection = client.getorcreatecollection(
            name="knowledgebase",

            embeddingfunction=embeddingfn,

            metadata={"description": "Knowledge base dari website dokumentasi"}
        )

        # Batch insert
        batchsize = 50
        for i in range(0, len(chunks), batchsize):

            batch = chunks[i:i + batchsize]

            collection.add(
                ids=[c["chunkid"] for c in batch],

                documents=[c["chunktext"] for c in batch],
                metadatas=[{
                    "url": c["url"],
                    "title": c["title"],
                    "docid": c["docid"]
                } for c in batch]
            )

            print(f"  Inserted batch {i // batchsize + 1}: "

                  f"{len(batch)} chunks")

        print(f"Total {len(chunks)} chunks disimpan ke ChromaDB")
        return collection

    def queryknowledgebase(self, collection, query, nresults=5):
        """Query knowledge base."""
        results = collection.query(
            querytexts=[query],

            nresults=nresults

        )

        print(f"Query: '{query}'")
        print(f"Hasil ({len(results['documents'][0])}):\n")

        for i, (doc, meta, dist) in enumerate(zip(
            results["documents"][0],
            results["metadatas"][0],
            results["distances"][0]
        )):
            print(f"  {i+1}. [{meta['title']}]({meta['url']})")
            print(f"     Distance: {dist:.4f}")
            print(f"     Preview: {doc[:150]}...")
            print()

        return results

    def savedocuments(self, outputpath):

        """Simpan dokumen ke file JSON."""
        with open(outputpath, 'w', encoding='utf-8') as f:
            json.dump(self.documents, f, ensureascii=False, indent=2)

        print(f"Dokumen disimpan ke {outputpath}")

Pipeline lengkap
def runknowledgebasepipeline():

    """Jalankan pipeline knowledge base secara lengkap."""

    builder = KnowledgeBaseBuilder(
        firecrawlapikey="fc-YOURAPIKEY"

    )

    # Step 1: Crawl website dokumentasi
    documents = builder.crawlwebsite(
        "https://docs.example.com/",
        limit=50,
        maxdepth=3

    )

    # Step 2: Simpan dokumen mentah
    builder.savedocuments("./knowledgebaseraw.json")

    # Step 3: Chunk dokumen
    chunks = builder.chunkdocuments(chunksize=1000, overlap=200)

    # Step 4: Simpan ke vector store
    collection = builder.storetochromadb(chunks)

    # Step 5: Test query
    builder.queryknowledgebase(
        collection,
        "Bagaimana cara mengkonfigurasi authentication?"
    )

    print("\nKnowledge base berhasil dibangun!")

Jalankan
runknowledgebasepipeline()

from openai import OpenAI
import chromadb
from chromadb.utils import embeddingfunctions

def ragquery(question, collection, openaiclient, ncontext=5):

    """RAG query menggunakan knowledge base."""

    # Retrieve konteks dari knowledge base
    results = collection.query(
        querytexts=[question],
        nresults=ncontext
    )

    # Gabungkan konteks
    contextparts = []

    for doc, meta in zip(results["documents"][0], results["metadatas"][0]):
        contextparts.append(f"Source: {meta['title']} ({meta['url']})\n{doc}")

    context = "\n\n".join(contextparts)


    # Generate jawaban dengan LLM
    response = openaiclient.chat.completions.create(
        model="gpt-4",
        messages=[
            {
                "role": "system",
                "content": (
                    "Anda adalah asisten yang menjawab pertanyaan berdasarkan "
                    "konteks yang diberikan. Jawab dengan akurat berdasarkan "
                    "konteks. Jika informasi tidak ada dalam konteks, katakan "
                    "dengan jujur."
                )
            },
            {
                "role": "user",
                "content": f"Konteks:\n{context}\n\nPertanyaan: {question}"
            }
        ],
        temperature=0.1
    )

    answer = response.choices[0].message.content

    # Return jawaban dengan sumber
    sources = [
        {"title": m["title"], "url": m["url"]}
        for m in results["metadatas"][0]
    ]

    return {
        "answer": answer,
        "sources": sources
    }

Penggunaan
openaiclient = OpenAI(apikey="sk-...")
result = ragquery(

"Bagaimana cara setup authentication di aplikasi?",
collection,
openaiclient
)
print(result["answer"])
print("Sources:", result["sources"])

Tips dan Best Practices

Kesimpulan

Firecrawl menyederhanakan proses web scraping untuk kebutuhan AI dan LLM. Dengan output markdown yang bersih, kemampuan JavaScript rendering, dan fitur structured data extraction, Firecrawl menjadi alat yang sangat berguna untuk membangun pipeline data AI.

Kombinasi Firecrawl dengan vector database seperti ChromaDB menciptakan fondasi yang kuat untuk sistem RAG. Fitur webhook dan batch processing membuatnya cocok untuk pipeline produksi yang membutuhkan reliability dan skalabilitas.

Pilih antara managed API untuk kemudahan atau self-hosted untuk kontrol penuh, sesuai kebutuhan proyek dan kebijakan data organisasi Anda.