crawl

type Crawler interface {
	// Schedule - Schedules request.
	// Context is passed to queue in a job.
	Schedule(context.Context, *Request) error

	// Execute - Makes a http request respecting context deadline.
	// If request Raw is not true - ParseHTML() method is executed on Response.
	// Then all callbacks are executed with context.
	Execute(context.Context, *Request) (*Response, error)

	// Handlers - Returns all registered handlers.
	Handlers() map[string][]Handler

	// Register - Registers crawl handler.
	Register(name string, h Handler)

	// Middleware - Registers a middleware.
	// Request is not executed if middleware returns an error.
	Middleware(Middleware)

	// Start - Starts the crawler.
	// All errors should be received from Errors() channel.
	Start()

	// Close - Closes the queue and the crawler.
	Close() error

	// Errors - Returns channel that will receive all crawl errors.
	// Only errors from queued requests are here.
	// Not only request errors but also queue errors.
	Errors() <-chan error
}

type Finder interface {
	Find(string) *goquery.Selection
}

type Handler func(context.Context, *Response) error

type Job interface {
	// Request - Returns crawl job.
	Request() *Request

	// Context - Returns job context.
	Context() context.Context

	// Done - Sets job as done.
	Done()
}

type Middleware func(context.Context, *Request, *http.Request) error

type Option func(*crawl)

type Queue interface {
	// Get - Gets request from Queue channel.
	// Returns io.EOF if queue is empty.
	Get() (Job, error)

	// Schedule - Schedules a Request.
	// Returns io.ErrClosedPipe if queue is closed.
	Schedule(context.Context, *Request) error

	// Close - Closes the queue.
	Close() error
}

type Request struct {
	// URL - It can be absolute URL or a relative to source URL if referer is set.
	URL string `json:"url,omitempty"`
	// Method - "GET" by default.
	Method string `json:"method,omitempty"`
	// Referer - Request referer.
	Referer string `json:"referer,omitempty"`
	// Form - Form values which set as request body.
	Form url.Values `json:"form,omitempty"`
	// Query - Form values which set as url query.
	Query url.Values `json:"query,omitempty"`
	// Cookies - Request cookies.
	Cookies url.Values `json:"cookies,omitempty"`
	// Header - Header values.
	Header map[string]string `json:"header,omitempty"`
	// Raw - when set to false, it means we expect HTML response
	Raw bool `json:"raw,omitempty"`
	// Callbacks - Crawl callback list.
	Callbacks []string `json:"callbacks,omitempty"`
}

type RequestError struct {
	*Request
	Err error
}

type Response struct {
	*Request
	*http.Response
	// contains filtered or unexported fields
}