crawler

type Channels map[int]chan Page

type Config struct {
	StartURL            string
	AllowedDomains      []string // Domains to stay within
	UserAgents          []string
	CrawlDelay          time.Duration // Delay between requests to the same domain
	MaxDepth            int           // Maximum crawl depth
	MaxRetries          int           // Max retries for a failed request
	RequestTimeout      time.Duration
	QueueIdleTimeout    time.Duration
	ProxyURL            string // e.g., "http://user:pass@host:port"
	RobotsUserAgent     string // User agent to use for robots.txt checks
	ConcurrentRequests  int    // Number of concurrent fetch workers
	Channels            Channels
	Headers             map[string]string
	LanguageCode        string
	Filters             []func(*Page, *Config) bool
	MaxIdleConnsPerHost int
	MaxIdleConns        int
	Proxies             []string
	RequireHeadless     bool
	ProcessSitemaps     bool
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Headless struct {
}

type Page struct {
	URL  *url.URL      // Page url
	Resp *PageResponse // Page response as returned from the GET request
	Body string        // Response body string
}

type PageResponse struct {
	StatusCode int
	Headers    map[string]any
	Body       io.Reader
}