namespace LexWells.Infrastructure.Common.Models;

public record RobotsEntry(
    bool IsAllowed, 
    int? CrawlDelay, 
    string? SitemapUrl, 
    DateTime ExpiresAt);