Polars - Ultra-Fast DataFrame Library Complete Tutorial

Table of Contents

Introduction

Polars is a blazingly fast DataFrame library written in Rust with Python bindings. It leverages Apache Arrow's columnar memory format and a multi-threaded query engine to deliver performance that is often 10-100x faster than Pandas. Polars supports both eager and lazy evaluation, making it ideal for both exploratory data analysis and production data pipelines.

Key advantages of Polars over Pandas:

• Written in Rust with zero-copy interoperability via Apache Arrow
• Multi-threaded execution by default
• Lazy evaluation with query optimization
• Consistent API with no index-based confusion
• Streaming support for out-of-core processing

Prerequisites

• Python 3.8 or higher
• Basic understanding of DataFrames and tabular data

pip install polars
pip install polars[all]  # Includes optional dependencies (Excel, database connectors, etc.)
pip install pandas       # For comparison benchmarks
pip install scikit-learn # For ML pipeline examples

Polars Basics

Creating DataFrames

import polars as pl

From a dictionary
df = pl.DataFrame({
    "name": ["Alice", "Bob", "Charlie", "Diana", "Eve"],
    "age": [30, 25, 35, 28, 32],
    "department": ["Engineering", "Marketing", "Engineering", "Sales", "Marketing"],
    "salary": [95000, 65000, 105000, 72000, 78000],
    "joindate": ["2020-01-15", "2021-06-01", "2019-03-20", "2022-01-10", "2020-09-05"]

})

Cast date column
df = df.withcolumns(pl.col("joindate").str.todate())

print(df)
print(f"Shape: {df.shape}")
print(f"Schema: {df.schema}")
print(f"Dtypes: {df.dtypes}")

Reading Data

# CSV
df = pl.readcsv("data.csv")


Parquet (highly recommended for performance)
df = pl.readparquet("data.parquet")

JSON
df = pl.readjson("data.json")


From Pandas
import pandas as pd
pandasdf = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
polarsdf = pl.frompandas(pandasdf)


Back to Pandas when needed
pandasback = polarsdf.topandas()

Basic Operations

# Select columns
df.select("name", "salary")
df.select(pl.col("name"), pl.col("salary"))

Filter rows
df.filter(pl.col("salary") > 80000)
df.filter((pl.col("department") == "Engineering") & (pl.col("age") > 30))

Sort
df.sort("salary", descending=True)
df.sort(["department", "salary"], descending=[False, True])

Add/modify columns
df.withcolumns(

    (pl.col("salary")  1.1).alias("salaryafterraise"),

    pl.col("name").str.touppercase().alias("nameupper"),

    pl.lit("Active").alias("status")
)

Drop columns
df.drop("joindate")

Rename columns
df.rename({"name": "employeename", "salary": "annualsalary"})

Descriptive statistics
df.describe()

import polars as pl

EAGER: operations execute immediately
df = pl.readcsv("largefile.csv")
result = df.filter(pl.col("amount") > 100).select("id", "amount").sort("amount")

LAZY: builds a query plan, optimizes, then executes
result = (
    pl.scancsv("largefile.csv")       # Returns LazyFrame
    .filter(pl.col("amount") > 100)
    .select("id", "amount")
    .sort("amount")
    .collect()                           # Triggers execution
)

Inspect the query plan
lazyquery = (

    pl.scancsv("largefile.csv")

    .filter(pl.col("amount") > 100)
    .select("id", "amount")
    .sort("amount")
)

See the optimized plan
print(lazyquery.explain())              # Optimized plan
print(lazyquery.explain(optimized=False))  # Unoptimized plan

# Example: The optimizer will only read "id" and "amount" columns from the CSV
and apply the filter during reading, not after loading the entire file
result = (
    pl.scancsv("largefile.csv")

    .filter(pl.col("amount") > 100)
    .select("id", "amount")
    .collect()
)

import polars as pl

df = pl.DataFrame({
    "product": ["A", "B", "A", "C", "B", "A", "C", "B"],
    "quantity": [10, 20, 15, 5, 25, 30, 10, 15],
    "price": [9.99, 14.99, 9.99, 24.99, 14.99, 9.99, 24.99, 14.99],
    "region": ["North", "South", "North", "East", "South", "East", "North", "East"]
})

Column expressions
result = df.select(
    pl.col("product"),
    pl.col("quantity"),
    (pl.col("quantity")  pl.col("price")).alias("revenue"),
    pl.col("quantity").sum().alias("totalquantity"),
    pl.col("quantity").mean().alias("avgquantity"),

    pl.col("quantity").std().alias("stdquantity"),
)

String expressions
dfstrings = pl.DataFrame({

    "text": ["Hello World", "foo bar baz", "POLARS IS FAST", "data science"]
})

result = dfstrings.select(
    pl.col("text"),
    pl.col("text").str.tolowercase().alias("lower"),

    pl.col("text").str.touppercase().alias("upper"),
    pl.col("text").str.split(" ").alias("words"),
    pl.col("text").str.lenchars().alias("charcount"),
    pl.col("text").str.contains("(?i)world|fast").alias("matchespattern"),

    pl.col("text").str.replaceall(" ", "").alias("underscored"),

)

Conditional expressions (when/then/otherwise)
result = df.withcolumns(
    pl.when(pl.col("quantity") > 20)
    .then(pl.lit("High"))
    .when(pl.col("quantity") > 10)
    .then(pl.lit("Medium"))
    .otherwise(pl.lit("Low"))
    .alias("demandlevel")

)

List/Array expressions
dflists = pl.DataFrame({
    "tags": [["python", "rust"], ["java", "python"], ["rust", "go", "python"]],
    "scores": [[90, 85, 92], [78, 88], [95, 91, 87, 93]]
})

result = dflists.select(

    pl.col("tags").list.len().alias("numtags"),
    pl.col("tags").list.contains("python").alias("haspython"),

    pl.col("scores").list.mean().alias("avgscore"),
    pl.col("scores").list.max().alias("maxscore"),

    pl.col("scores").list.sort().alias("sortedscores"),
)

Date/Time expressions
dfdates = pl.DataFrame({

    "timestamp": pl.daterange(pl.date(2024, 1, 1), pl.date(2024, 12, 31), eager=True),
}).withcolumns(

    pl.col("timestamp").dt.year().alias("year"),
    pl.col("timestamp").dt.month().alias("month"),
    pl.col("timestamp").dt.weekday().alias("weekday"),
    pl.col("timestamp").dt.quarter().alias("quarter"),
    pl.col("timestamp").dt.isleapyear().alias("isleap"),
)

Joins and Combining DataFrames

# Sample DataFrames
orders = pl.DataFrame({
    "orderid": [1, 2, 3, 4, 5],

    "customerid": [101, 102, 101, 103, 104],
    "productid": ["P1", "P2", "P3", "P1", "P2"],

    "amount": [250.0, 150.0, 300.0, 250.0, 150.0]
})

customers = pl.DataFrame({
    "customerid": [101, 102, 103, 105],
    "name": ["Alice", "Bob", "Charlie", "Eve"],
    "tier": ["Gold", "Silver", "Gold", "Bronze"]
})

products = pl.DataFrame({
    "productid": ["P1", "P2", "P3"],

    "productname": ["Widget", "Gadget", "Doohickey"],
    "category": ["Electronics", "Electronics", "Tools"]
})

Inner join
result = orders.join(customers, on="customerid", how="inner")


Left join
result = orders.join(customers, on="customerid", how="left")

Multiple joins chained
result = (
    orders
    .join(customers, on="customerid", how="left")

    .join(products, on="productid", how="left")
)

Join with different column names
dfa = pl.DataFrame({"ida": [1, 2, 3], "valuea": [10, 20, 30]})

dfb = pl.DataFrame({"idb": [1, 2, 4], "valueb": [100, 200, 400]})
result = dfa.join(dfb, lefton="ida", righton="idb", how="outer")

Cross join
sizes = pl.DataFrame({"size": ["S", "M", "L"]})
colors = pl.DataFrame({"color": ["Red", "Blue"]})
combinations = sizes.join(colors, how="cross")

Semi join (keep rows from left that have a match in right)
activecustomers = orders.join(customers, on="customerid", how="semi")

Anti join (keep rows from left that do NOT have a match in right)
ordersmissingcustomer = orders.join(customers, on="customerid", how="anti")


Concatenation
df1 = pl.DataFrame({"a": [1, 2], "b": [3, 4]})
df2 = pl.DataFrame({"a": [5, 6], "b": [7, 8]})
combined = pl.concat([df1, df2])  # Vertical (row-wise)

Horizontal concatenation
combinedh = pl.concat([df1, df2], how="horizontal")

Group By and Aggregations

sales = pl.DataFrame({
    "date": ["2024-01-15", "2024-01-15", "2024-02-10", "2024-02-10",
             "2024-03-05", "2024-03-05", "2024-01-20", "2024-02-25"],
    "product": ["A", "B", "A", "B", "A", "B", "C", "C"],
    "region": ["North", "South", "North", "South", "North", "South", "East", "East"],
    "quantity": [100, 150, 200, 120, 180, 90, 50, 75],
    "revenue": [1000, 2250, 2000, 1800, 1800, 1350, 1250, 1875]
})

Simple group by
result = sales.groupby("product").agg(

    pl.col("quantity").sum().alias("totalquantity"),
    pl.col("revenue").sum().alias("totalrevenue"),

    pl.col("revenue").mean().alias("avgrevenue"),
    pl.len().alias("numtransactions"),

)

Multiple group by columns
result = sales.groupby("product", "region").agg(
    pl.col("quantity").sum().alias("totalquantity"),

    pl.col("revenue").sum().alias("totalrevenue"),
)

Advanced aggregations
result = sales.groupby("product").agg(

    pl.col("quantity").sum().alias("totalqty"),
    pl.col("revenue").sum().alias("totalrev"),

    pl.col("revenue").min().alias("minrev"),
    pl.col("revenue").max().alias("maxrev"),

    pl.col("revenue").std().alias("stdrev"),
    pl.col("revenue").quantile(0.75).alias("q75rev"),

    pl.col("region").nunique().alias("regionsserved"),

    pl.col("date").first().alias("firstsale"),
    pl.col("date").last().alias("lastsale"),

)

Group by with sorting
result = (
    sales
    .groupby("product")
    .agg(pl.col("revenue").sum().alias("totalrevenue"))

    .sort("totalrevenue", descending=True)
)

Dynamic group by (time-based)
salests = sales.withcolumns(pl.col("date").str.todate())

monthly = salests.groupbydynamic("date", every="1mo").agg(
    pl.col("revenue").sum().alias("monthlyrevenue"),

    pl.col("quantity").sum().alias("monthlyquantity"),
)

Window Functions

Window functions compute values across rows related to the current row without collapsing the DataFrame:

df = pl.DataFrame({
    "employee": ["Alice", "Bob", "Charlie", "Diana", "Eve", "Frank"],
    "department": ["Eng", "Eng", "Eng", "Sales", "Sales", "Sales"],
    "salary": [95000, 85000, 105000, 72000, 78000, 68000],
    "hireyear": [2020, 2021, 2019, 2022, 2020, 2023]

})

over() - Polars' window function
result = df.withcolumns(
    # Average salary within each department
    pl.col("salary").mean().over("department").alias("deptavgsalary"),

    # Rank within department by salary
    pl.col("salary").rank(descending=True).over("department").alias("salaryrank"),


    # Min and max salary in department
    pl.col("salary").min().over("department").alias("deptminsalary"),

    pl.col("salary").max().over("department").alias("deptmaxsalary"),


    # Count of employees in department
    pl.len().over("department").alias("deptsize"),

    # Difference from department average
    (pl.col("salary") - pl.col("salary").mean().over("department")).alias("salaryvsavg"),
)

Rolling window functions
tsdata = pl.DataFrame({

    "date": pl.daterange(pl.date(2024, 1, 1), pl.date(2024, 3, 31), eager=True),
    "value": [float(i) + (i % 7)  3.0 for i in range(91)]

})

result = tsdata.withcolumns(
    pl.col("value").rollingmean(windowsize=7).alias("rolling7davg"),
    pl.col("value").rollingsum(windowsize=7).alias("rolling7dsum"),
    pl.col("value").rollingstd(windowsize=7).alias("rolling7dstd"),
    pl.col("value").rollingmin(windowsize=14).alias("rolling14dmin"),
    pl.col("value").rollingmax(windowsize=14).alias("rolling14dmax"),
    pl.col("value").shift(1).alias("prevvalue"),

    (pl.col("value") - pl.col("value").shift(1)).alias("dailychange"),
    ((pl.col("value") / pl.col("value").shift(1)) - 1).alias("dailypctchange"),
)

Cumulative operations
result = df.sort("salary").withcolumns(

    pl.col("salary").cumsum().alias("cumulativesalary"),

    pl.col("salary").cummax().alias("runningmax"),

    pl.col("salary").cummin().alias("runningmin"),

    pl.col("salary").cumcount().alias("runningcount"),

)

import polars as pl
import pandas as pd
import numpy as np
import time

def benchmarkcomparison(nrows: int = 10000000):

    # Generate test data
    np.random.seed(42)
    data = {
        "id": np.arange(nrows),
        "category": np.random.choice(["A", "B", "C", "D", "E"], nrows),

        "value1": np.random.randn(nrows),
        "value2": np.random.randn(nrows),

        "value3": np.random.uniform(0, 1000, nrows),
    }

    # Pandas
    pdf = pd.DataFrame(data)
    start = time.time()
    pdfresult = (

        pdf.groupby("category")
        .agg({"value1": "mean", "value2": "sum", "value3": ["min", "max", "std"]})
    )
    pandastime = time.time() - start

    # Polars
    pldf = pl.DataFrame(data)
    start = time.time()
    plresult = pldf.groupby("category").agg(
        pl.col("value1").mean(),
        pl.col("value2").sum(),
        pl.col("value3").min().alias("value3min"),

        pl.col("value3").max().alias("value3max"),
        pl.col("value3").std().alias("value3std"),

    )
    polarstime = time.time() - start

    print(f"Data size: {nrows:,} rows")

    print(f"Pandas:    {pandastime:.3f}s")
    print(f"Polars:    {polarstime:.3f}s")

    print(f"Speedup:   {pandastime / polarstime:.1f}x")


    # Filter benchmark
    start = time.time()
     = pdf[(pdf["value1"] > 0) & (pdf["value3"] < 500)]
    pandasfilter = time.time() - start


    start = time.time()
     = pldf.filter((pl.col("value1") > 0) & (pl.col("value3") < 500))
    polarsfilter = time.time() - start


    print(f"\nFilter benchmark:")
    print(f"Pandas:    {pandasfilter:.3f}s")
    print(f"Polars:    {polarsfilter:.3f}s")

    print(f"Speedup:   {pandasfilter / polarsfilter:.1f}x")


    # Sort benchmark
    start = time.time()
     = pdf.sortvalues(["category", "value1"])

    pandassort = time.time() - start

    start = time.time()
     = pldf.sort(["category", "value1"])

    polarssort = time.time() - start

    print(f"\nSort benchmark:")
    print(f"Pandas:    {pandassort:.3f}s")

    print(f"Polars:    {polarssort:.3f}s")
    print(f"Speedup:   {pandassort / polarssort:.1f}x")

benchmarkcomparison(10000000)

import polars as pl
import numpy as np
from sklearn.modelselection import traintestsplit
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classificationreport


Load and prepare data with Polars
df = pl.readcsv("customerdata.csv")


Feature engineering with Polars
dffeatures = df.withcolumns(

    # Numerical transformations
    pl.col("purchaseamount").log().alias("logpurchase"),

    (pl.col("purchaseamount") / pl.col("visits")).alias("avgpurchasepervisit"),

    pl.col("dayssincelastpurchase").clip(0, 365).alias("recencycapped"),


    # Categorical encoding
    pl.col("region").cast(pl.Categorical).alias("regioncat"),

    # Time-based features
    pl.col("signupdate").str.todate().dt.year().alias("signupyear"),

    pl.col("signupdate").str.todate().dt.month().alias("signupmonth"),

    # Group-based features
    pl.col("purchaseamount").mean().over("region").alias("regionavgpurchase"),

    pl.col("purchaseamount").rank().over("region").alias("purchaserankinregion"),

)

Define features and target
featurecols = [
    "logpurchase", "avgpurchasepervisit", "recencycapped",

    "signupyear", "signupmonth", "regionavgpurchase",

    "purchaserankinregion", "visits", "age"
]

Convert to numpy for sklearn
X = dffeatures.select(featurecols).tonumpy()

y = dffeatures.select("churn").tonumpy().flatten()


Split data
Xtrain, Xtest, ytrain, ytest = traintestsplit(X, y, testsize=0.2, randomstate=42)


Scale features
scaler = StandardScaler()
Xtrainscaled = scaler.fittransform(Xtrain)

Xtestscaled = scaler.transform(Xtest)

Train model
clf = RandomForestClassifier(nestimators=100, randomstate=42, njobs=-1)

clf.fit(Xtrainscaled, ytrain)

Evaluate
ypred = clf.predict(Xtestscaled)

print(classificationreport(ytest, ypred))

Feature importance back in Polars
importancedf = pl.DataFrame({

    "feature": featurecols,
    "importance": clf.featureimportances
}).sort("importance", descending=True)

print(importancedf)

import polars as pl

Streaming: process data in chunks without loading everything into memory
result = (
    pl.scancsv("hugedataset.csv")  # Does not load into memory

    .filter(pl.col("status") == "active")
    .groupby("category")
    .agg(
        pl.col("amount").sum().alias("totalamount"),

        pl.col("amount").mean().alias("avgamount"),
        pl.len().alias("count")
    )
    .sort("totalamount", descending=True)

    .collect(streaming=True)  # Process in streaming mode
)

Scan Parquet files (even more efficient)
result = (
    pl.scanparquet("data/.parquet")  # Glob pattern for multiple files
    .filter(pl.col("year") >= 2023)
    .select("id", "category", "amount", "year")
    .collect(streaming=True)
)

Sink results directly to file (never materializes full result in memory)
(
    pl.scancsv("inputlarge.csv")
    .filter(pl.col("value") > 0)
    .withcolumns(

        (pl.col("value")  2).alias("doubledvalue")
    )
    .sinkparquet("outputprocessed.parquet")
)

Process multiple large files efficiently
import glob

lazyframes = [pl.scanparquet(f) for f in glob.glob("data/part.parquet")]
combined = pl.concat(lazyframes)

result = (
    combined
    .groupby("customerid")
    .agg(
        pl.col("purchaseamount").sum().alias("lifetimevalue"),
        pl.col("purchasedate").max().alias("lastpurchase"),
        pl.len().alias("totalorders")

    )
    .filter(pl.col("lifetimevalue") > 1000)
    .sort("lifetimevalue", descending=True)

    .head(1000)
    .collect(streaming=True)
)

Convert large CSV to Parquet for better performance
(
    pl.scancsv("legacydata.csv")

    .sinkparquet(
        "legacydata.parquet",

        compression="zstd",
        rowgroupsize=100000
    )
)

Best Practices

Conclusion

Polars represents a significant advancement in the Python data processing ecosystem. Its Rust-based engine, lazy evaluation with query optimization, and seamless streaming support make it the ideal choice for both analytical workloads and production data pipelines. By adopting Polars best practices -- lazy evaluation, Parquet file formats, columnar thinking, and streaming -- you can process datasets orders of magnitude faster than with traditional tools, while using significantly less memory. For new projects, Polars should be your default DataFrame library.