Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
- API
  
  Reference documentation for the pkg.go.dev API
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

grawlr

package module

Go to main page

Versions in this module

v0

v0.1.1

Apr 6, 2025

v0.1.0

Apr 5, 2025

Changes in this version

+ var ErrDepthLimitExceeded = func(depth, limit int) error

+ var ErrForbiddenURL = func(u string) error

+ var ErrRobotsDisallowed = func(u string) error

+ var ErrVisitedURL = func(u string) error

+ type Harvester struct

+ AllowRevisit bool

+ AllowedURLs []string

+ Client *http.Client

+ Context context.Context

+ DepthLimit int

+ DisallowedURLs []string

+ func NewHarvester(options ...Options) *Harvester

+ func (h *Harvester) Clone() *Harvester

+ func (h *Harvester) HtmlDo(gqSelector string, fn HtmlCallback)

+ func (h *Harvester) RequestDo(mw ReqMiddleware)

+ func (h *Harvester) ResponseDo(mw ResMiddleware)

+ func (h *Harvester) Visit(u string) error

+ type HtmlCallback func(el *HtmlElement)

+ type HtmlElement struct

+ Request *Request

+ Response *Response

+ Selection *goquery.Selection

+ Text string

+ func (e *HtmlElement) Attribute(key string) string

+ type HtmlMiddleware struct

+ Function HtmlCallback

+ Selector string

+ type InMemoryStore struct

+ func NewInMemoryStore() *InMemoryStore

+ func (s *InMemoryStore) Visit(url string)

+ func (s *InMemoryStore) Visited(url string) bool

+ type Options func(h *Harvester)

+ func WithAllowRevisit(allow bool) Options

+ func WithAllowedURLs(urls []string) Options

+ func WithClient(client *http.Client) Options

+ func WithContext(ctx context.Context) Options

+ func WithDepthLimit(depth int) Options

+ func WithDisallowedURLs(urls []string) Options

+ func WithIgnoreRobots(ignore bool) Options

+ func WithStore(store Storer) Options

+ type ReqMiddleware func(req *Request)

+ type Request struct

+ BaseURL *url.URL

+ Body io.Reader

+ Depth int

+ Headers *http.Header

+ Host string

+ Method string

+ URL *url.URL

+ func (r *Request) GetAbsoluteURL(link string) string

+ func (r *Request) Visit(u string) error

+ type ResMiddleware func(res *Response)

+ type Response struct

+ Body io.Reader

+ Headers *http.Header

+ Request *Request

+ StatusCode int

+ type Storer interface

+ Visit func(url string)

+ Visited func(url string) bool

v0.1.0-alpha

Apr 5, 2025