models

package
v0.7.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 26, 2025 License: MIT Imports: 7 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var FULL_REFRESH bool
View Source
var STREAM_NAME string
View Source
var StateMutex sync.RWMutex

Functions

This section is empty.

Types

type AuthConfig

type AuthConfig struct {
	Required bool            `json:"required,omitempty"`
	Strategy string          `json:"strategy,omitempty"`
	Basic    BasicAuthConfig `json:"basic,omitempty"`
	Token    TokenAuthConfig `json:"token,omitempty"`
	OAuth    OAuthConfig     `json:"oauth,omitempty"`
}

type BasicAuthConfig

type BasicAuthConfig struct {
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
}

type Bookmark

type Bookmark struct {
	UpdatedAt string            `json:"updated_at"`
	Latest    map[string]string `json:"latest"`
}

type Message

type Message struct {
	Type               string                 `json:"type"`
	Record             map[string]interface{} `json:"record,omitempty"`
	Stream             string                 `json:"stream,omitempty"`
	Schema             interface{}            `json:"schema,omitempty"`
	Value              interface{}            `json:"value,omitempty"`
	KeyProperties      []string               `json:"key_properties,omitempty"`
	BookmarkProperties []string               `json:"bookmark_properties,omitempty"`
	Required           []string               `json:"required,omitempty"`
}

type OAuthConfig

type OAuthConfig struct {
	ClientID     string `json:"client_id,omitempty"`
	ClientSecret string `json:"client_secret,omitempty"`
	RefreshToken string `json:"refresh_token,omitempty"`
	TokenURL     string `json:"token_url,omitempty"`
}

type PaginationQueryConfig

type PaginationQueryConfig struct {
	QueryParameter string `json:"query_parameter,omitempty"`
	QueryValue     int    `json:"query_value,omitempty"`
	QueryIncrement int    `json:"query_increment,omitempty"`
}

type RecordsConfig

type RecordsConfig struct {
	UniqueKeyPath       []string   `json:"unique_key_path,omitempty"`
	DropFieldPaths      [][]string `json:"drop_field_paths,omitempty"`
	SensitiveFieldPaths [][]string `json:"sensitive_field_paths,omitempty"`
}

type ResponseConfig

type ResponseConfig struct {
	RecordsPath        []string              `json:"records_path,omitempty"`
	Pagination         bool                  `json:"pagination,omitempty"`
	PaginationStrategy string                `json:"pagination_strategy,omitempty"`
	PaginationNextPath []string              `json:"pagination_next_path,omitempty"`
	PaginationQuery    PaginationQueryConfig `json:"pagination_query,omitempty"`
}

type RestConfig

type RestConfig struct {
	Auth     AuthConfig     `json:"auth,omitempty"`
	Response ResponseConfig `json:"response,omitempty"`
}

type StreamCatalog

type StreamCatalog struct {
	KeyProperties []string               `json:"key_properties"`
	Schema        map[string]interface{} `json:"schema"`
	Stream        string                 `json:"stream"`
}
var DerivedCatalog StreamCatalog

func (*StreamCatalog) Create

func (c *StreamCatalog) Create() error

Create <STREAM>_catalog.json

func (*StreamCatalog) Message

func (c *StreamCatalog) Message() error

Message generates a schema message from the derived catalog

func (*StreamCatalog) Read

func (c *StreamCatalog) Read() error

Read the Catalog JSON file

func (*StreamCatalog) RecordVersusCatalog

func (c *StreamCatalog) RecordVersusCatalog(record map[string]interface{}) (bool, error)

RecordVersusCatalog validates record against Catalog

func (*StreamCatalog) Update

func (c *StreamCatalog) Update() error

Update the Catalog JSON file

type StreamConfig

type StreamConfig struct {
	StreamName     string        `json:"stream_name,omitempty"`
	SourceType     string        `json:"source_type,omitempty"`
	URL            string        `json:"url,omitempty"`
	MaxConcurrency int           `json:"max_concurrency,omitempty"`
	Records        RecordsConfig `json:"records,omitempty"`
	Rest           RestConfig    `json:"rest,omitempty"`
}
var Config StreamConfig

type StreamState

type StreamState struct {
	Stream   string   `json:"stream"`
	Bookmark Bookmark `json:"bookmark"`
}
var State StreamState

func (*StreamState) Create

func (s *StreamState) Create() error

Create creates a state JSON file for the stream

func (*StreamState) Message

func (s *StreamState) Message() error

Message generates a message with the current state

func (*StreamState) Read

func (s *StreamState) Read() error

Reads State JSON file

func (*StreamState) Update

func (s *StreamState) Update(record map[string]interface{})

Updates the State JSON file

type TokenAuthConfig

type TokenAuthConfig struct {
	Header      string `json:"header,omitempty"`
	HeaderValue string `json:"header_value,omitempty"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL