EagleCast

internal/manager/manager.go

@@ -0,0 +1,827 @@
+package manager
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"html/template"
+	"log"
+	"net/textproto"
+	"net/url"
+	"path"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"maps"
+
+	"github.com/Masterminds/sprig/v3"
+	"source.offmarket.win/aleagle/eaglecast/internal/i18n"
+	"source.offmarket.win/aleagle/eaglecast/internal/notifs"
+	"source.offmarket.win/aleagle/eaglecast/models"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+)
+
+// attribInlineEmbed is the HTML attrib used to mark <img> tags whose source
+// should be embedded as a multipart cid inline attachment.
+const attribInlineEmbed = "data-embed"
+
+var (
+	reInlineImage = regexp.MustCompile(`(?is)<img\b[^>]*\b` + attribInlineEmbed + `\b[^>]*>`)
+	reImgSrc      = regexp.MustCompile(`(?is)(\s)src\s*=\s*(?:"([^"]*)"|'([^']*)')`)
+)
+
+const (
+	// BaseTPL is the name of the base template.
+	BaseTPL = "base"
+
+	// ContentTpl is the name of the compiled message.
+	ContentTpl = "content"
+
+	dummyUUID = "00000000-0000-0000-0000-000000000000"
+)
+
+// Store represents a data backend, such as a database,
+// that provides subscriber and campaign records.
+type Store interface {
+	NextCampaigns(currentIDs []int64, sentCounts []int64) ([]*models.Campaign, error)
+	NextSubscribers(campID, limit int) ([]models.Subscriber, error)
+	GetCampaign(campID int) (*models.Campaign, error)
+	GetAttachment(mediaID int) (models.Attachment, error)
+	GetInlineAttachmentByFilename(filename string) (models.Attachment, string, error)
+	UpdateCampaignStatus(campID int, status string) error
+	UpdateCampaignCounts(campID int, toSend int, sent int, lastSubID int) error
+	CreateLink(url string) (string, error)
+	BlocklistSubscriber(id int64) error
+	DeleteSubscriber(id int64) error
+}
+
+// Messenger is an interface for a generic messaging backend,
+// for instance, e-mail, SMS etc.
+type Messenger interface {
+	Name() string
+	Push(models.Message) error
+	Flush() error
+	Close() error
+}
+
+// CampStats contains campaign stats like per minute send rate.
+type CampStats struct {
+	SendRate int
+}
+
+// Manager handles the scheduling, processing, and queuing of campaigns
+// and message pushes.
+type Manager struct {
+	cfg        Config
+	store      Store
+	i18n       *i18n.I18n
+	messengers map[string]Messenger
+	fnNotify   func(subject string, data any) error
+	log        *log.Logger
+
+	// Campaigns that are currently running.
+	pipes    map[int]*pipe
+	pipesMut sync.RWMutex
+
+	tpls    map[int]*models.Template
+	tplsMut sync.RWMutex
+
+	// Links generated using Track() are cached here so as to not query
+	// the database for the link UUID for every message sent. This has to
+	// be locked as it may be used externally when previewing campaigns.
+	links    map[string]string
+	linksMut sync.RWMutex
+
+	nextPipes chan *pipe
+	campMsgQ  chan CampaignMessage
+	msgQ      chan models.Message
+
+	// Sliding window keeps track of the total number of messages sent in a period
+	// and on reaching the specified limit, waits until the window is over before
+	// sending further messages.
+	slidingCount int
+	slidingStart time.Time
+
+	tplFuncs template.FuncMap
+}
+
+// CampaignMessage represents an instance of campaign message to be pushed out,
+// specific to a subscriber, via the campaign's messenger.
+type CampaignMessage struct {
+	Campaign   *models.Campaign
+	Subscriber models.Subscriber
+
+	from     string
+	to       string
+	subject  string
+	body     []byte
+	altBody  []byte
+	unsubURL string
+	headers  models.Headers
+
+	pipe *pipe
+}
+
+// Config has parameters for configuring the manager.
+type Config struct {
+	// Number of subscribers to pull from the DB in a single iteration.
+	BatchSize             int
+	Concurrency           int
+	MessageRate           int
+	MaxSendErrors         int
+	SlidingWindow         bool
+	SlidingWindowDuration time.Duration
+	SlidingWindowRate     int
+	RequeueOnError        bool
+	FromEmail             string
+	IndividualTracking    bool
+	DisableTracking       bool
+	LinkTrackURL          string
+	UnsubURL              string
+	OptinURL              string
+	MessageURL            string
+	ViewTrackURL          string
+	ArchiveURL            string
+	RootURL               string
+	UnsubHeader           bool
+
+	// Interval to scan the DB for active campaign checkpoints.
+	ScanInterval time.Duration
+
+	// ScanCampaigns indicates whether this instance of manager will scan the DB
+	// for active campaigns and process them.
+	// This can be used to run multiple instances of eaglecast
+	// (exposed to the internet, private etc.) where only one does campaign
+	// processing while the others handle other kinds of traffic.
+	ScanCampaigns bool
+}
+
+var pushTimeout = time.Second * 3
+
+// New returns a new instance of Mailer.
+func New(cfg Config, store Store, i *i18n.I18n, l *log.Logger) *Manager {
+	if cfg.BatchSize < 1 {
+		cfg.BatchSize = 1000
+	}
+	if cfg.Concurrency < 1 {
+		cfg.Concurrency = 1
+	}
+	if cfg.MessageRate < 1 {
+		cfg.MessageRate = 1
+	}
+
+	m := &Manager{
+		cfg:   cfg,
+		store: store,
+		i18n:  i,
+		fnNotify: func(subject string, data any) error {
+			return notifs.NotifySystem(subject, notifs.TplCampaignStatus, data, nil)
+		},
+		log:          l,
+		messengers:   make(map[string]Messenger),
+		pipes:        make(map[int]*pipe),
+		tpls:         make(map[int]*models.Template),
+		links:        make(map[string]string),
+		nextPipes:    make(chan *pipe, 1000),
+		campMsgQ:     make(chan CampaignMessage, cfg.Concurrency*cfg.MessageRate*2),
+		msgQ:         make(chan models.Message, cfg.Concurrency*cfg.MessageRate*2),
+		slidingStart: time.Now(),
+	}
+	m.tplFuncs = m.makeGnericFuncMap()
+
+	return m
+}
+
+// AddMessenger adds a Messenger messaging backend to the manager.
+func (m *Manager) AddMessenger(msg Messenger) error {
+	id := msg.Name()
+	if _, ok := m.messengers[id]; ok {
+		return fmt.Errorf("messenger '%s' is already loaded", id)
+	}
+	m.messengers[id] = msg
+
+	return nil
+}
+
+// PushMessage pushes an arbitrary non-campaign Message to be sent out by the workers.
+// It times out if the queue is busy.
+func (m *Manager) PushMessage(msg models.Message) error {
+	t := time.NewTicker(pushTimeout)
+	defer t.Stop()
+
+	select {
+	case m.msgQ <- msg:
+	case <-t.C:
+		m.log.Printf("message push timed out: '%s'", msg.Subject)
+		return errors.New("message push timed out")
+	}
+
+	return nil
+}
+
+// PushCampaignMessage pushes a campaign messages into a queue to be sent out by the workers.
+// It times out if the queue is busy.
+func (m *Manager) PushCampaignMessage(msg CampaignMessage) error {
+	t := time.NewTicker(pushTimeout)
+	defer t.Stop()
+
+	// Load any media/attachments.
+	if err := m.attachMedia(msg.Campaign); err != nil {
+		return err
+	}
+
+	select {
+	case m.campMsgQ <- msg:
+	case <-t.C:
+		m.log.Printf("message push timed out: '%s'", msg.Subject())
+		return errors.New("message push timed out")
+	}
+
+	return nil
+}
+
+// HasMessenger checks if a given messenger is registered.
+func (m *Manager) HasMessenger(id string) bool {
+	_, ok := m.messengers[id]
+
+	return ok
+}
+
+// HasRunningCampaigns checks if there are any active campaigns.
+func (m *Manager) HasRunningCampaigns() bool {
+	m.pipesMut.Lock()
+	defer m.pipesMut.Unlock()
+
+	return len(m.pipes) > 0
+}
+
+// GetCampaignStats returns campaign statistics.
+func (m *Manager) GetCampaignStats(id int) CampStats {
+	n := 0
+
+	m.pipesMut.Lock()
+	if c, ok := m.pipes[id]; ok {
+		n = int(c.rate.Rate())
+	}
+	m.pipesMut.Unlock()
+
+	return CampStats{SendRate: n}
+}
+
+// Run is a blocking function (that should be invoked as a goroutine)
+// that scans the data source at regular intervals for pending campaigns,
+// and queues them for processing. The process queue fetches batches of
+// subscribers and pushes messages to them for each queued campaign
+// until all subscribers are exhausted, at which point, a campaign is marked
+// as "finished".
+func (m *Manager) Run() {
+	if m.cfg.ScanCampaigns {
+		// Periodically scan campaigns and push running campaigns to nextPipes
+		// to fetch subscribers from the campaign.
+		go m.scanCampaigns(m.cfg.ScanInterval)
+	}
+
+	// Spawn N message workers.
+	for i := 0; i < m.cfg.Concurrency; i++ {
+		go m.worker()
+	}
+
+	// Indefinitely wait on the pipe queue to fetch the next set of subscribers
+	// for any active campaigns.
+	for p := range m.nextPipes {
+		has, err := p.NextSubscribers()
+		if err != nil {
+			m.log.Printf("error processing campaign batch (%s): %v", p.camp.Name, err)
+
+			// If the batch fails, stop the pipe and release it so that it doesn't hang forever.
+			// The cleanup() records the state in DB and scanCampaigns() picks it up at a later point.
+			p.Stop(false)
+			p.wg.Done()
+			continue
+		}
+
+		if has {
+			// There are more subscribers to fetch. Queue again.
+			select {
+			case m.nextPipes <- p:
+			default:
+				// If the queue is full for any reason, stop the pipe and release it.
+				// The cleanup() records the state in DB and scanCampaigns() picks it up
+				// at a later point.
+				p.Stop(false)
+				p.wg.Done()
+			}
+		} else {
+			// The pipe is created with a +1 on the waitgroup pseudo counter
+			// so that it immediately waits. Subsequently, every message created
+			// is incremented in the counter in pipe.newMessage(), and when it's'
+			// processed (or ignored when a campaign is paused or cancelled),
+			// the count is's reduced in worker().
+			//
+			// This marks down the original non-message +1, causing the waitgroup
+			// to be released and the pipe to end, triggering the pg.Wait()
+			// in newPipe() that calls pipe.cleanup().
+			p.wg.Done()
+		}
+	}
+}
+
+// CacheTpl caches a template for ad-hoc use. This is currently only used by tx templates.
+func (m *Manager) CacheTpl(id int, tpl *models.Template) {
+	if body, atts := m.ApplyInlineImages(tpl.Body); len(atts) > 0 {
+		tpl.Body = body
+		tpl.Attachments = atts
+		if err := tpl.Compile(m.GenericTemplateFuncs()); err != nil {
+			m.log.Printf("error recompiling tx template %d after inline image: %v", id, err)
+			return
+		}
+	}
+
+	m.tplsMut.Lock()
+	m.tpls[id] = tpl
+	m.tplsMut.Unlock()
+}
+
+// DeleteTpl deletes a cached template.
+func (m *Manager) DeleteTpl(id int) {
+	m.tplsMut.Lock()
+	delete(m.tpls, id)
+	m.tplsMut.Unlock()
+}
+
+// GetTpl returns a cached template.
+func (m *Manager) GetTpl(id int) (*models.Template, error) {
+	m.tplsMut.RLock()
+	tpl, ok := m.tpls[id]
+	m.tplsMut.RUnlock()
+
+	if !ok {
+		return nil, fmt.Errorf("template %d not found", id)
+	}
+
+	return tpl, nil
+}
+
+// TemplateFuncs returns the template functions to be applied into
+// compiled campaign templates.
+func (m *Manager) TemplateFuncs(c *models.Campaign) template.FuncMap {
+	f := template.FuncMap{
+		"TrackLink": func(url string, msg *CampaignMessage) string {
+			if m.cfg.DisableTracking {
+				return url
+			}
+
+			subUUID := msg.Subscriber.UUID
+			if !m.cfg.IndividualTracking {
+				subUUID = dummyUUID
+			}
+
+			return m.trackLink(url, msg.Campaign.UUID, subUUID)
+		},
+		"TrackView": func(msg *CampaignMessage) template.HTML {
+			if m.cfg.DisableTracking {
+				return template.HTML("")
+			}
+
+			subUUID := msg.Subscriber.UUID
+			if !m.cfg.IndividualTracking {
+				subUUID = dummyUUID
+			}
+
+			return template.HTML(fmt.Sprintf(`<img src="%s" alt="" />`,
+				fmt.Sprintf(m.cfg.ViewTrackURL, msg.Campaign.UUID, subUUID)))
+		},
+		"UnsubscribeURL": func(msg *CampaignMessage) string {
+			return msg.unsubURL
+		},
+		"ManageURL": func(msg *CampaignMessage) string {
+			return msg.unsubURL + "?manage=true"
+		},
+		"OptinURL": func(msg *CampaignMessage) string {
+			// Add list IDs.
+			// TODO: Show private lists list on optin e-mail
+			return fmt.Sprintf(m.cfg.OptinURL, msg.Subscriber.UUID, "")
+		},
+		"MessageURL": func(msg *CampaignMessage) string {
+			return fmt.Sprintf(m.cfg.MessageURL, c.UUID, msg.Subscriber.UUID)
+		},
+		"ArchiveURL": func() string {
+			return m.cfg.ArchiveURL
+		},
+		"RootURL": func() string {
+			return m.cfg.RootURL
+		},
+	}
+
+	maps.Copy(f, m.tplFuncs)
+
+	return f
+}
+
+func (m *Manager) GenericTemplateFuncs() template.FuncMap {
+	return m.tplFuncs
+}
+
+// StopCampaign marks a running campaign as stopped so that all its queued messages are ignored.
+func (m *Manager) StopCampaign(id int) {
+	m.pipesMut.RLock()
+	if p, ok := m.pipes[id]; ok {
+		p.Stop(false)
+	}
+	m.pipesMut.RUnlock()
+}
+
+// Close closes and exits the campaign manager.
+func (m *Manager) Close() {
+	close(m.nextPipes)
+	close(m.msgQ)
+}
+
+// scanCampaigns is a blocking function that periodically scans the data source
+// for campaigns to process and dispatches them to the manager. It feeds campaigns
+// into nextPipes.
+func (m *Manager) scanCampaigns(tick time.Duration) {
+	t := time.NewTicker(tick)
+	defer t.Stop()
+
+	// Periodically scan the data source for campaigns to process.
+	for range t.C {
+		ids, counts := m.getCurrentCampaigns()
+		campaigns, err := m.store.NextCampaigns(ids, counts)
+		if err != nil {
+			m.log.Printf("error fetching campaigns: %v", err)
+			continue
+		}
+
+		for _, c := range campaigns {
+			// Create a new pipe that'll handle this campaign's states.
+			p, err := m.newPipe(c)
+			if err != nil {
+				m.log.Printf("error processing campaign (%s): %v", c.Name, err)
+				continue
+			}
+			m.log.Printf("start processing campaign (%s)", c.Name)
+
+			// If subscriber processing is busy, move on. Blocking and waiting
+			// can end up in a race condition where the waiting campaign's
+			// state in the data source has changed.
+			select {
+			case m.nextPipes <- p:
+			default:
+				// If the queue is full for any reason, stop the pipe and release it.
+				// The cleanup() records the state in DB and scanCampaigns() picks it up
+				// at a later point.
+				p.Stop(false)
+				p.wg.Done()
+			}
+		}
+	}
+}
+
+// worker is a blocking function that perpetually listents to events (message) on different
+// queues and processes them.
+func (m *Manager) worker() {
+	// Counter to keep track of the message / sec rate limit.
+	numMsg := 0
+	for {
+		select {
+		// Campaign message.
+		case msg, ok := <-m.campMsgQ:
+			if !ok {
+				return
+			}
+
+			// If the campaign has ended or stopped, ignore the message.
+			if msg.pipe != nil && msg.pipe.stopped.Load() {
+				// Reduce the message counter on the pipe.
+				msg.pipe.wg.Done()
+				continue
+			}
+
+			// Pause on hitting the message rate.
+			if numMsg >= m.cfg.MessageRate {
+				time.Sleep(time.Second)
+				numMsg = 0
+			}
+			numMsg++
+
+			// Outgoing message.
+			out := models.Message{
+				From:        msg.from,
+				To:          []string{msg.to},
+				Subject:     msg.subject,
+				ContentType: msg.Campaign.ContentType,
+				Body:        msg.body,
+				AltBody:     msg.altBody,
+				Subscriber:  msg.Subscriber,
+				Campaign:    msg.Campaign,
+				Attachments: msg.Campaign.Attachments,
+			}
+
+			h := textproto.MIMEHeader{}
+			h.Set(models.EmailHeaderCampaignUUID, msg.Campaign.UUID)
+			h.Set(models.EmailHeaderSubscriberUUID, msg.Subscriber.UUID)
+
+			// Attach List-Unsubscribe headers?
+			if m.cfg.UnsubHeader {
+				h.Set("List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
+				h.Set("List-Unsubscribe", `<`+msg.unsubURL+`>`)
+			}
+
+			// Attach any custom headers.
+			for _, set := range msg.headers {
+				for hdr, val := range set {
+					h.Add(hdr, val)
+				}
+			}
+
+			// Set the headers.
+			out.Headers = h
+
+			// Push the message to the messenger.
+			err := m.messengers[msg.Campaign.Messenger].Push(out)
+			if err != nil {
+				m.log.Printf("error sending message in campaign %s: subscriber %d: %v", msg.Campaign.Name, msg.Subscriber.ID, err)
+			}
+
+			// Increment the send rate or the error counter if there was an error.
+			if msg.pipe != nil {
+				// Mark the message as done.
+				msg.pipe.wg.Done()
+
+				if err != nil {
+					// Call the error callback, which keeps track of the error count
+					// and stops the campaign if the error count exceeds the threshold.
+					msg.pipe.OnError()
+				} else {
+					id := uint64(msg.Subscriber.ID)
+					if id > msg.pipe.lastID.Load() {
+						msg.pipe.lastID.Store(uint64(msg.Subscriber.ID))
+					}
+					msg.pipe.rate.Incr(1)
+					msg.pipe.sent.Add(1)
+				}
+			}
+
+		// Arbitrary message.
+		case msg, ok := <-m.msgQ:
+			if !ok {
+				return
+			}
+
+			// Push the message to the messenger.
+			if err := m.messengers[msg.Messenger].Push(msg); err != nil {
+				m.log.Printf("error sending message '%s': %v", msg.Subject, err)
+			}
+		}
+	}
+}
+
+// getCurrentCampaigns returns the IDs of campaigns currently being processed
+// and their sent counts.
+func (m *Manager) getCurrentCampaigns() ([]int64, []int64) {
+	// Needs to return an empty slice in case there are no campaigns.
+	m.pipesMut.RLock()
+	defer m.pipesMut.RUnlock()
+
+	var (
+		ids    = make([]int64, 0, len(m.pipes))
+		counts = make([]int64, 0, len(m.pipes))
+	)
+	for _, p := range m.pipes {
+		ids = append(ids, int64(p.camp.ID))
+
+		// Get the sent counts for campaigns and reset them to 0
+		// as in the database, they're stored cumulatively (sent += $newSent).
+		counts = append(counts, p.sent.Load())
+		p.sent.Store(0)
+	}
+
+	return ids, counts
+}
+
+// trackLink register a URL and return its UUID to be used in message templates
+// for tracking links.
+func (m *Manager) trackLink(url, campUUID, subUUID string) string {
+	if m.cfg.DisableTracking {
+		return url
+	}
+
+	url = strings.ReplaceAll(url, "&amp;", "&")
+
+	m.linksMut.RLock()
+	if uu, ok := m.links[url]; ok {
+		m.linksMut.RUnlock()
+		return fmt.Sprintf(m.cfg.LinkTrackURL, uu, campUUID, subUUID)
+	}
+	m.linksMut.RUnlock()
+
+	// Register link.
+	uu, err := m.store.CreateLink(url)
+	if err != nil {
+		m.log.Printf("error registering tracking for link '%s': %v", url, err)
+
+		// If the registration fails, fail over to the original URL.
+		return url
+	}
+
+	m.linksMut.Lock()
+	m.links[url] = uu
+	m.linksMut.Unlock()
+
+	return fmt.Sprintf(m.cfg.LinkTrackURL, uu, campUUID, subUUID)
+}
+
+// sendNotif sends a notification to registered admin e-mails.
+func (m *Manager) sendNotif(c *models.Campaign, status, reason string) error {
+	var (
+		subject = fmt.Sprintf("%s: %s", cases.Title(language.Und).String(status), c.Name)
+		data    = map[string]any{
+			"ID":     c.ID,
+			"Name":   c.Name,
+			"Status": status,
+			"Sent":   c.Sent,
+			"ToSend": c.ToSend,
+			"Reason": reason,
+		}
+	)
+
+	return m.fnNotify(subject, data)
+}
+
+// makeGnericFuncMap returns a generic template func map with custom template
+// functions and sprig template functions.
+func (m *Manager) makeGnericFuncMap() template.FuncMap {
+	funcs := template.FuncMap{
+		"Date": func(layout string) string {
+			if layout == "" {
+				layout = time.ANSIC
+			}
+			return time.Now().Format(layout)
+		},
+		"L": func() *i18n.I18n {
+			return m.i18n
+		},
+		"Safe": func(safeHTML string) template.HTML {
+			return template.HTML(safeHTML)
+		},
+	}
+
+	// Copy spring functions.
+	sprigFuncs := sprig.GenericFuncMap()
+	delete(sprigFuncs, "env")
+	delete(sprigFuncs, "expandenv")
+	delete(sprigFuncs, "getHostByName")
+
+	maps.Copy(funcs, sprigFuncs)
+
+	return funcs
+}
+
+// attachMedia loads any media/attachments from the media store and attaches
+// the byte blobs to the campaign. Inline attachments are skipped as they're
+// loaded earlier by LoadInlineImages().
+func (m *Manager) attachMedia(c *models.Campaign) error {
+	// Already loaded if any non-inline attachment is present.
+	for _, a := range c.Attachments {
+		if !a.IsInline {
+			return nil
+		}
+	}
+
+	for _, mid := range []int64(c.MediaIDs) {
+		a, err := m.store.GetAttachment(int(mid))
+		if err != nil {
+			return fmt.Errorf("error fetching attachment %d on campaign %s: %v", mid, c.Name, err)
+		}
+		c.Attachments = append(c.Attachments, a)
+	}
+
+	return nil
+}
+
+// LoadInlineImages resolves any <img ... data-embed ...> tags in the campaign
+// body and template body one time before CompileTemplate.
+func (m *Manager) LoadInlineImages(c *models.Campaign) error {
+	if c.ContentType == models.CampaignContentTypePlain {
+		return nil
+	}
+
+	cidCache := make(map[string]string)
+	body, atts := m.applyInlineImages(c.Body, cidCache)
+	c.Body = body
+
+	tplBody, tplAtts := m.applyInlineImages(c.TemplateBody, cidCache)
+	c.TemplateBody = tplBody
+	atts = append(atts, tplAtts...)
+
+	c.Attachments = append(c.Attachments, atts...)
+	return nil
+}
+
+// ApplyInlineImages scans body for <img ... data-embed ...> tags, resolves
+// each unique src filename to a media item, attaches it as an inline part, and
+// rewrites the matched img src to cid.
+func (m *Manager) ApplyInlineImages(body string) (string, []models.Attachment) {
+	return m.applyInlineImages(body, make(map[string]string))
+}
+
+func (m *Manager) applyInlineImages(body string, cache map[string]string) (string, []models.Attachment) {
+	if !strings.Contains(body, attribInlineEmbed) {
+		return body, nil
+	}
+
+	var atts []models.Attachment
+	out := reInlineImage.ReplaceAllStringFunc(body, func(tag string) string {
+		src := extractSrc(tag)
+		if src == "" || strings.HasPrefix(strings.ToLower(src), "cid:") {
+			return tag
+		}
+
+		fname := filenameFromSrc(src)
+		if fname == "" {
+			return tag
+		}
+
+		cid, ok := cache[src]
+		if !ok {
+			if a, c, err := m.store.GetInlineAttachmentByFilename(fname); err == nil {
+				atts = append(atts, a)
+				cid = c
+			} else {
+				m.log.Printf("inline image %q not embedded: %v", src, err)
+			}
+			cache[src] = cid
+		}
+		if cid == "" {
+			return tag
+		}
+		return reImgSrc.ReplaceAllString(tag, `${1}src="cid:`+cid+`"`)
+	})
+
+	return out, atts
+}
+
+// MakeContentID returns a standard `Content-ID` value (without  the angle brackets).
+func MakeContentID(key string) string {
+	sum := sha1.Sum([]byte(key))
+	return hex.EncodeToString(sum[:8]) + "@email"
+}
+
+// filenameFromSrc extracts a media filename from an <img src> value (ignoring
+// any URLs and just capturing the filename.
+func filenameFromSrc(src string) string {
+	if src == "" {
+		return ""
+	}
+	p := src
+	if u, err := url.Parse(src); err == nil && u.Path != "" {
+		p = u.Path
+	}
+	return path.Base(p)
+}
+
+// MakeAttachmentHeader returns a textproto.MIMEHeader for an email
+// attachment. Default encoding is base64  and contentType is application/octet-stream.
+func MakeAttachmentHeader(filename, encoding, contentType string) textproto.MIMEHeader {
+	return makeAttachmentHeader("attachment", filename, encoding, contentType, "")
+}
+
+// MakeInlineAttachmentHeader returns a textproto.MIMEHeader for an inline
+// image attachment referenced from the HTML body via a cid URL.
+func MakeInlineAttachmentHeader(filename, encoding, contentType, contentID string) textproto.MIMEHeader {
+	return makeAttachmentHeader("inline", filename, encoding, contentType, contentID)
+}
+
+func makeAttachmentHeader(disposition, filename, encoding, contentType, contentID string) textproto.MIMEHeader {
+	if encoding == "" {
+		encoding = "base64"
+	}
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+
+	h := textproto.MIMEHeader{}
+	h.Set("Content-Disposition", disposition+"; filename="+filename)
+	h.Set("Content-Type", fmt.Sprintf("%s; name=\""+filename+"\"", contentType))
+	h.Set("Content-Transfer-Encoding", encoding)
+	if contentID != "" {
+		h.Set("Content-ID", "<"+contentID+">")
+	}
+	return h
+}
+
+func extractSrc(tag string) string {
+	m := reImgSrc.FindStringSubmatch(tag)
+	if len(m) == 0 {
+		return ""
+	}
+	if m[2] != "" {
+		return m[2]
+	}
+	return m[3]
+}

internal/manager/message.go

@@ -0,0 +1,107 @@
+package manager
+
+import (
+	"bytes"
+	"fmt"
+
+	"source.offmarket.win/aleagle/eaglecast/models"
+)
+
+// NewCampaignMessage creates and returns a CampaignMessage that is made available
+// to message templates while they're compiled. It represents a message from
+// a campaign that's bound to a single Subscriber.
+func (m *Manager) NewCampaignMessage(c *models.Campaign, s models.Subscriber) (CampaignMessage, error) {
+	msg := CampaignMessage{
+		Campaign:   c,
+		Subscriber: s,
+
+		subject:  c.Subject,
+		from:     c.FromEmail,
+		to:       s.Email,
+		unsubURL: fmt.Sprintf(m.cfg.UnsubURL, c.UUID, s.UUID),
+	}
+
+	if err := msg.render(); err != nil {
+		return msg, err
+	}
+
+	return msg, nil
+}
+
+// render takes a Message, executes its pre-compiled Campaign.Tpl
+// and applies the resultant bytes to Message.body to be used in messages.
+func (m *CampaignMessage) render() error {
+	out := bytes.Buffer{}
+
+	// Render the subject if it's a template.
+	if m.Campaign.SubjectTpl != nil {
+		if err := m.Campaign.SubjectTpl.ExecuteTemplate(&out, models.ContentTpl, m); err != nil {
+			return err
+		}
+		m.subject = out.String()
+		out.Reset()
+	}
+
+	// Compile the main template.
+	if err := m.Campaign.Tpl.ExecuteTemplate(&out, models.BaseTpl, m); err != nil {
+		return err
+	}
+	m.body = out.Bytes()
+
+	// Is there an alt body?
+	if m.Campaign.ContentType != models.CampaignContentTypePlain && m.Campaign.AltBody.Valid {
+		if m.Campaign.AltBodyTpl != nil {
+			b := bytes.Buffer{}
+			if err := m.Campaign.AltBodyTpl.ExecuteTemplate(&b, models.ContentTpl, m); err != nil {
+				return err
+			}
+			m.altBody = b.Bytes()
+		} else {
+			m.altBody = []byte(m.Campaign.AltBody.String)
+		}
+	}
+
+	// If there are templated headers, compile them.
+	if m.Campaign.HeaderTpls == nil {
+		m.headers = m.Campaign.Headers
+	} else {
+		hdrOut := bytes.Buffer{}
+		m.headers = make(models.Headers, len(m.Campaign.Headers))
+		for i, set := range m.Campaign.Headers {
+			m.headers[i] = make(map[string]string, len(set))
+			for hdr, val := range set {
+				tpl := m.Campaign.HeaderTpls[i][hdr]
+				if tpl == nil {
+					m.headers[i][hdr] = val
+					continue
+				}
+				hdrOut.Reset()
+				if err := tpl.ExecuteTemplate(&hdrOut, models.ContentTpl, m); err != nil {
+					return fmt.Errorf("error rendering header %q: %v", hdr, err)
+				}
+				m.headers[i][hdr] = hdrOut.String()
+			}
+		}
+	}
+
+	return nil
+}
+
+// Subject returns a copy of the message subject
+func (m *CampaignMessage) Subject() string {
+	return m.subject
+}
+
+// Body returns a copy of the message body.
+func (m *CampaignMessage) Body() []byte {
+	out := make([]byte, len(m.body))
+	copy(out, m.body)
+	return out
+}
+
+// AltBody returns a copy of the message's alt body.
+func (m *CampaignMessage) AltBody() []byte {
+	out := make([]byte, len(m.altBody))
+	copy(out, m.altBody)
+	return out
+}

internal/manager/pipe.go

@@ -0,0 +1,244 @@
+package manager
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"source.offmarket.win/aleagle/eaglecast/models"
+	"github.com/paulbellamy/ratecounter"
+)
+
+type pipe struct {
+	camp       *models.Campaign
+	rate       *ratecounter.RateCounter
+	wg         *sync.WaitGroup
+	sent       atomic.Int64
+	lastID     atomic.Uint64
+	errors     atomic.Uint64
+	stopped    atomic.Bool
+	withErrors atomic.Bool
+
+	m *Manager
+}
+
+// newPipe adds a campaign to the process queue.
+func (m *Manager) newPipe(c *models.Campaign) (*pipe, error) {
+	// Validate messenger.
+	if _, ok := m.messengers[c.Messenger]; !ok {
+		m.store.UpdateCampaignStatus(c.ID, models.CampaignStatusCancelled)
+		return nil, fmt.Errorf("unknown messenger %s on campaign %s", c.Messenger, c.Name)
+	}
+
+	// Resolve any inline images before compiling the template.
+	if err := m.LoadInlineImages(c); err != nil {
+		return nil, err
+	}
+
+	// Load the template.
+	if err := c.CompileTemplate(m.TemplateFuncs(c)); err != nil {
+		return nil, err
+	}
+
+	// Load any media/attachments.
+	if err := m.attachMedia(c); err != nil {
+		return nil, err
+	}
+
+	// Add the campaign to the active map.
+	p := &pipe{
+		camp: c,
+		rate: ratecounter.NewRateCounter(time.Minute),
+		wg:   &sync.WaitGroup{},
+		m:    m,
+	}
+
+	// Increment the waitgroup so that Wait() blocks immediately. This is necessary
+	// as a campaign pipe is created first and subscribers/messages under it are
+	// fetched asynchronolusly later. The messages each add to the wg and that
+	// count is used to determine the exhaustion/completion of all messages.
+	p.wg.Add(1)
+
+	go func() {
+		// Wait for all the messages in the campaign to be processed
+		// (successfully or skipped after errors or cancellation).
+		p.wg.Wait()
+
+		p.cleanup()
+	}()
+
+	m.pipesMut.Lock()
+	m.pipes[c.ID] = p
+	m.pipesMut.Unlock()
+	return p, nil
+}
+
+// NextSubscribers processes the next batch of subscribers in a given campaign.
+// It returns a bool indicating whether any subscribers were processed
+// in the current batch or not. A false indicates that all subscribers
+// have been processed, or that a campaign has been paused or cancelled.
+func (p *pipe) NextSubscribers() (bool, error) {
+	// Fetch the next batch of subscribers from a 'running' campaign.
+	subs, err := p.m.store.NextSubscribers(p.camp.ID, p.m.cfg.BatchSize)
+	if err != nil {
+		return false, fmt.Errorf("error fetching campaign subscribers (%s): %v", p.camp.Name, err)
+	}
+
+	// There are no subscribers from the query. Either all subscribers on the campaign
+	// have been processed, or the campaign has changed from 'running' to 'paused' or 'cancelled'.
+	if len(subs) == 0 {
+		return false, nil
+	}
+
+	// Is there a sliding window limit configured?
+	hasSliding := p.m.cfg.SlidingWindow &&
+		p.m.cfg.SlidingWindowRate > 0 &&
+		p.m.cfg.SlidingWindowDuration.Seconds() > 1
+
+	// Push messages.
+	for _, s := range subs {
+		msg, err := p.newMessage(s)
+		if err != nil {
+			p.m.log.Printf("error rendering message (%s) (%s): %v", p.camp.Name, s.Email, err)
+			continue
+		}
+
+		// Push the message to the queue while blocking and waiting until
+		// the queue is drained.
+		p.m.campMsgQ <- msg
+
+		// Check if the sliding window is active.
+		if hasSliding {
+			diff := time.Since(p.m.slidingStart)
+
+			// Window has expired. Reset the clock.
+			if diff >= p.m.cfg.SlidingWindowDuration {
+				p.m.slidingStart = time.Now()
+				p.m.slidingCount = 0
+			}
+
+			// Have the messages exceeded the limit?
+			p.m.slidingCount++
+			if p.m.slidingCount >= p.m.cfg.SlidingWindowRate {
+				wait := p.m.cfg.SlidingWindowDuration - diff
+
+				p.m.log.Printf("messages exceeded (%d) for the window (%v since %s). Sleeping for %s.",
+					p.m.slidingCount,
+					p.m.cfg.SlidingWindowDuration,
+					p.m.slidingStart.Format(time.RFC822Z),
+					wait.Round(time.Second)*1)
+
+				p.m.slidingCount = 0
+				time.Sleep(wait)
+			}
+		}
+	}
+
+	return true, nil
+}
+
+// OnError keeps track of the number of errors that occur while sending messages
+// and pauses the campaign if the error threshold is met.
+func (p *pipe) OnError() {
+	if p.m.cfg.MaxSendErrors < 1 {
+		return
+	}
+
+	// If the error threshold is met, pause the campaign.
+	count := p.errors.Add(1)
+	if int(count) < p.m.cfg.MaxSendErrors {
+		return
+	}
+
+	p.Stop(true)
+	p.m.log.Printf("error count exceeded %d. pausing campaign %s", p.m.cfg.MaxSendErrors, p.camp.Name)
+}
+
+// Stop "marks" a campaign as stopped. It doesn't actually stop the processing
+// of messages. That happens when every queued message in the campaign is processed,
+// marking .wg, the waitgroup counter as done. That triggers cleanup().
+func (p *pipe) Stop(withErrors bool) {
+	// Already stopped.
+	if p.stopped.Load() {
+		return
+	}
+
+	if withErrors {
+		p.withErrors.Store(true)
+	}
+
+	p.stopped.Store(true)
+}
+
+// newMessage returns a campaign message while internally incrementing the
+// number of messages in the pipe wait group so that the status of every
+// message can be atomically tracked.
+func (p *pipe) newMessage(s models.Subscriber) (CampaignMessage, error) {
+	msg, err := p.m.NewCampaignMessage(p.camp, s)
+	if err != nil {
+		return msg, err
+	}
+
+	msg.pipe = p
+	p.wg.Add(1)
+
+	return msg, nil
+}
+
+// cleanup finishes the campaign and updates the campaign status in the DB
+// and also triggers a notification to the admin. This only triggers once
+// a pipe's wg counter is fully exhausted, draining all messages in its queue.
+func (p *pipe) cleanup() {
+	defer func() {
+		p.m.pipesMut.Lock()
+		delete(p.m.pipes, p.camp.ID)
+		p.m.pipesMut.Unlock()
+	}()
+
+	// Update campaign's 'sent count.
+	if err := p.m.store.UpdateCampaignCounts(p.camp.ID, 0, int(p.sent.Load()), int(p.lastID.Load())); err != nil {
+		p.m.log.Printf("error updating campaign counts (%s): %v", p.camp.Name, err)
+	}
+
+	// The campaign was auto-paused due to errors.
+	if p.withErrors.Load() {
+		if err := p.m.store.UpdateCampaignStatus(p.camp.ID, models.CampaignStatusPaused); err != nil {
+			p.m.log.Printf("error updating campaign (%s) status to %s: %v", p.camp.Name, models.CampaignStatusPaused, err)
+		} else {
+			p.m.log.Printf("set campaign (%s) to %s", p.camp.Name, models.CampaignStatusPaused)
+		}
+
+		_ = p.m.sendNotif(p.camp, models.CampaignStatusPaused, "Too many errors")
+		return
+	}
+
+	// The campaign was manually stopped (pause, cancel).
+	if p.stopped.Load() {
+		p.m.log.Printf("stop processing campaign (%s)", p.camp.Name)
+		return
+	}
+
+	// Campaign wasn't manually stopped and subscribers were naturally exhausted.
+	// Fetch the up-to-date campaign status from the DB.
+	c, err := p.m.store.GetCampaign(p.camp.ID)
+	if err != nil {
+		p.m.log.Printf("error fetching campaign (%s) for ending: %v", p.camp.Name, err)
+		return
+	}
+
+	// If a running campaign has exhausted subscribers, it's finished.
+	if c.Status == models.CampaignStatusRunning || c.Status == models.CampaignStatusScheduled {
+		c.Status = models.CampaignStatusFinished
+		if err := p.m.store.UpdateCampaignStatus(p.camp.ID, models.CampaignStatusFinished); err != nil {
+			p.m.log.Printf("error finishing campaign (%s): %v", p.camp.Name, err)
+		} else {
+			p.m.log.Printf("campaign (%s) finished", p.camp.Name)
+		}
+	} else {
+		p.m.log.Printf("finish processing campaign (%s)", p.camp.Name)
+	}
+
+	// Notify admin.
+	_ = p.m.sendNotif(c, c.Status, "")
+}