namespace LexWells.Infrastructure.Common.Interfaces;

public interface IRobotsService
{
    /// <summary>
    /// Checks if the User-Agent is allowed to access the specific path.
    /// </summary>
    Task<bool> CanCrawlAsync(Uri uri, string userAgent = "LexWellsBot");

    /// <summary>
    /// Gets the crawl delay specified by the host, or a default if none exists.
    /// </summary>
    Task<TimeSpan> GetCrawlDelayAsync(Uri uri, string userAgent = "LexWellsBot");
}