14 lines
473 B
C#
14 lines
473 B
C#
namespace LexWells.Infrastructure.Common.Interfaces;
|
|
|
|
public interface IRobotsService
|
|
{
|
|
/// <summary>
|
|
/// Checks if the User-Agent is allowed to access the specific path.
|
|
/// </summary>
|
|
Task<bool> CanCrawlAsync(Uri uri, string userAgent = "LexWellsBot");
|
|
|
|
/// <summary>
|
|
/// Gets the crawl delay specified by the host, or a default if none exists.
|
|
/// </summary>
|
|
Task<TimeSpan> GetCrawlDelayAsync(Uri uri, string userAgent = "LexWellsBot");
|
|
} |