28 — Exercises & Project Ideas

package crawler

import (
    "context"
    "net/http"
    "net/url"
    "sync"
    "time"
)

type Crawler struct {
    client    *http.Client
    workers   int
    maxDepth  int
    visited   sync.Map  // URL → struct{}
    rateLimit time.Duration
}

type Result struct {
    URL   string
    Title string
    Depth int
    Err   error
}

func New(workers, maxDepth int, rateLimit time.Duration) *Crawler {
    return &Crawler{
        client:    &http.Client{Timeout: 10 * time.Second},
        workers:   workers,
        maxDepth:  maxDepth,
        rateLimit: rateLimit,
    }
}

func (c *Crawler) Crawl(ctx context.Context, seed string) <-chan Result {
    out := make(chan Result, 100)
    jobs := make(chan job, 100)

    var wg sync.WaitGroup
    for i := 0; i < c.workers; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            c.worker(ctx, jobs, out)
        }()
    }

    go func() {
        defer close(jobs)
        c.enqueue(ctx, jobs, seed, 0)
    }()

    go func() {
        wg.Wait()
        close(out)
    }()

    return out
}

type job struct {
    url   string
    depth int
}

func (c *Crawler) enqueue(ctx context.Context, jobs chan<- job, u string, depth int) {
    if depth > c.maxDepth {
        return
    }
    if _, loaded := c.visited.LoadOrStore(u, struct{}{}); loaded {
        return  // already visited
    }
    select {
    case jobs <- job{url: u, depth: depth}:
    case <-ctx.Done():
    }
}

func (c *Crawler) worker(ctx context.Context, jobs <-chan job, out chan<- Result) {
    for {
        select {
        case j, ok := <-jobs:
            if !ok {
                return
            }
            result, links := c.fetch(ctx, j)
            select {
            case out <- result:
            case <-ctx.Done():
                return
            }
            for _, link := range links {
                c.enqueue(ctx, jobs, link, j.depth+1)
            }
        case <-ctx.Done():
            return
        }
    }
}

func (c *Crawler) fetch(ctx context.Context, j job) (Result, []string) {
    req, err := http.NewRequestWithContext(ctx, "GET", j.url, nil)
    if err != nil {
        return Result{URL: j.url, Depth: j.depth, Err: err}, nil
    }
    resp, err := c.client.Do(req)
    if err != nil {
        return Result{URL: j.url, Depth: j.depth, Err: err}, nil
    }
    defer resp.Body.Close()

    title, links := parseHTML(resp.Body, j.url)
    return Result{URL: j.url, Title: title, Depth: j.depth}, links
}

11. JSON Streaming Server

Build an HTTP server that streams large JSON responses (NDJSON) without buffering. Use json.Encoder, flusher.Flush() for chunked transfer.

12. Benchmark-Driven Optimization

Write a function that parses a custom log format. Benchmark it, profile with pprof, and reduce allocations to zero. Use sync.Pool, strings.Builder, and pre-allocation.

13. Graceful HTTP Server

Build an HTTP server that shuts down gracefully on SIGINT/SIGTERM — drains in-flight requests, closes idle connections, and stops accepting new ones. Use signal.NotifyContext and http.Server.Shutdown.

package main

import (
    "context"
    "log"
    "net/http"
    "os/signal"
    "syscall"
    "time"
)

func main() {
    srv := &http.Server{
        Addr:         ":8080",
        Handler:      mux(),
        ReadTimeout:  5 * time.Second,
        WriteTimeout: 10 * time.Second,
        IdleTimeout:  120 * time.Second,
    }

    ctx, stop := signal.NotifyContext(context.Background(),
        syscall.SIGINT, syscall.SIGTERM)
    defer stop()

    go func() {
        log.Printf("listening on %s", srv.Addr)
        if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
            log.Fatalf("listen: %v", err)
        }
    }()

    <-ctx.Done()
    log.Println("shutdown signal received")

    shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    if err := srv.Shutdown(shutdownCtx); err != nil {
        log.Printf("forced shutdown: %v", err)
    }
    log.Println("server stopped")
}