Merge pull request #696 from rapidpro/upmerge

Upmerge from Nyaruka

CHANGELOG.md

@@ -1,3 +1,12 @@
+v8.0.0 (2023-01-09)
+-------------------------
+ * Update test database to latest schema
+
+v7.5.36 (2023-01-02)
+-------------------------
+ * Update to latest goflow which adds locale field to MsgOut
+ * Improve error reporting when courier call fails
+
 v7.5.35 (2022-12-05)
 -------------------------
  * Retry messages which fail to queue to courier

core/models/msgs.go

@@ -418,7 +418,23 @@ func buildMsgMetadata(m *flows.MsgOut) map[string]interface{} {
 		metadata["quick_replies"] = m.QuickReplies()
 	}
 	if m.Templating() != nil {
-		metadata["templating"] = m.Templating()
+		mLanguage, mCountry := m.Locale().ToParts()
+
+		// TODO once we're queuing messages with locale and courier is reading that, can just add templating directly
+		// without language and country
+		metadata["templating"] = struct {
+			Template  *assets.TemplateReference `json:"template"`
+			Language  envs.Language             `json:"language"`
+			Country   envs.Country              `json:"country"`
+			Variables []string                  `json:"variables,omitempty"`
+			Namespace string                    `json:"namespace"`
+		}{
+			Template:  m.Templating_.Template(),
+			Language:  mLanguage,
+			Country:   mCountry,
+			Variables: m.Templating().Variables(),
+			Namespace: m.Templating().Namespace(),
+		}
 	}
 	if m.Topic() != flows.NilMsgTopic {
 		metadata["topic"] = string(m.Topic())
@@ -1017,12 +1033,14 @@ func (b *BroadcastBatch) CreateMessages(ctx context.Context, rt *runtime.Runtime
 
 		// not found? try org default language
 		if t == nil {
-			t = trans[oa.Env().DefaultLanguage()]
+			lang = oa.Env().DefaultLanguage()
+			t = trans[lang]
 		}
 
 		// not found? use broadcast base language
 		if t == nil {
-			t = trans[b.BaseLanguage]
+			lang = b.BaseLanguage
+			t = trans[lang]
 		}
 
 		if t == nil {
@@ -1066,7 +1084,7 @@ func (b *BroadcastBatch) CreateMessages(ctx context.Context, rt *runtime.Runtime
 		}
 
 		// create our outgoing message
-		out := flows.NewMsgOut(urn, channel.ChannelReference(), text, t.Attachments, t.QuickReplies, nil, flows.NilMsgTopic, unsendableReason)
+		out := flows.NewMsgOut(urn, channel.ChannelReference(), text, t.Attachments, t.QuickReplies, nil, flows.NilMsgTopic, envs.NewLocale(lang, envs.NilCountry), unsendableReason)
 		msg, err := NewOutgoingBroadcastMsg(rt, oa.Org(), channel, contact, out, time.Now(), b.BroadcastID)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating outgoing message")

core/models/msgs_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/nyaruka/mailroom/testsuite/testdata"
 	"github.com/nyaruka/null"
 	"github.com/nyaruka/redisx/assertredis"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -176,7 +175,7 @@ func TestNewOutgoingFlowMsg(t *testing.T) {
 			session.SetIncomingMsg(tc.ResponseTo, null.NullString)
 		}
 
-		flowMsg := flows.NewMsgOut(tc.URN, assets.NewChannelReference(tc.ChannelUUID, "Test Channel"), tc.Text, tc.Attachments, tc.QuickReplies, nil, tc.Topic, tc.Unsendable)
+		flowMsg := flows.NewMsgOut(tc.URN, assets.NewChannelReference(tc.ChannelUUID, "Test Channel"), tc.Text, tc.Attachments, tc.QuickReplies, nil, tc.Topic, envs.NilLocale, tc.Unsendable)
 		msg, err := models.NewOutgoingFlowMsg(rt, oa.Org(), channel, session, flow, flowMsg, now)
 
 		assert.NoError(t, err)
@@ -221,7 +220,7 @@ func TestNewOutgoingFlowMsg(t *testing.T) {
 
 	// check that msg loop detection triggers after 20 repeats of the same text
 	newOutgoing := func(text string) *models.Msg {
-		flowMsg := flows.NewMsgOut(urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", testdata.Cathy.URNID)), assets.NewChannelReference(testdata.TwilioChannel.UUID, "Twilio"), text, nil, nil, nil, flows.NilMsgTopic, flows.NilUnsendableReason)
+		flowMsg := flows.NewMsgOut(urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", testdata.Cathy.URNID)), assets.NewChannelReference(testdata.TwilioChannel.UUID, "Twilio"), text, nil, nil, nil, flows.NilMsgTopic, envs.NilLocale, flows.NilUnsendableReason)
 		msg, err := models.NewOutgoingFlowMsg(rt, oa.Org(), channel, session, flow, flowMsg, now)
 		require.NoError(t, err)
 		return msg
@@ -266,6 +265,7 @@ func TestMarshalMsg(t *testing.T) {
 		[]string{"yes", "no"},
 		nil,
 		flows.MsgTopicPurchase,
+		envs.NilLocale,
 		flows.NilUnsendableReason,
 	)
 
@@ -324,6 +324,7 @@ func TestMarshalMsg(t *testing.T) {
 		"Hi there",
 		nil, nil, nil,
 		flows.NilMsgTopic,
+		envs.NilLocale,
 		flows.NilUnsendableReason,
 	)
 	in1 := testdata.InsertIncomingMsg(db, testdata.Org1, testdata.TwilioChannel, testdata.Cathy, "test", models.MsgStatusHandled)
@@ -366,7 +367,7 @@ func TestMarshalMsg(t *testing.T) {
 
 	// try a broadcast message which won't have session and flow fields set
 	bcastID := testdata.InsertBroadcast(db, testdata.Org1, `eng`, map[envs.Language]string{`eng`: "Blast"}, models.NilScheduleID, []*testdata.Contact{testdata.Cathy}, nil)
-	bcastMsg1 := flows.NewMsgOut(urn, assets.NewChannelReference(testdata.TwilioChannel.UUID, "Test Channel"), "Blast", nil, nil, nil, flows.NilMsgTopic, flows.NilUnsendableReason)
+	bcastMsg1 := flows.NewMsgOut(urn, assets.NewChannelReference(testdata.TwilioChannel.UUID, "Test Channel"), "Blast", nil, nil, nil, flows.NilMsgTopic, envs.NilLocale, flows.NilUnsendableReason)
 	msg3, err := models.NewOutgoingBroadcastMsg(rt, oa.Org(), channel, cathy, bcastMsg1, time.Date(2021, 11, 9, 14, 3, 30, 0, time.UTC), bcastID)
 	require.NoError(t, err)
 
@@ -526,8 +527,8 @@ func TestGetMsgRepetitions(t *testing.T) {
 	oa := testdata.Org1.Load(rt)
 	_, cathy := testdata.Cathy.Load(db, oa)
 
-	msg1 := flows.NewMsgOut(testdata.Cathy.URN, nil, "foo", nil, nil, nil, flows.NilMsgTopic, flows.NilUnsendableReason)
-	msg2 := flows.NewMsgOut(testdata.Cathy.URN, nil, "bar", nil, nil, nil, flows.NilMsgTopic, flows.NilUnsendableReason)
+	msg1 := flows.NewMsgOut(testdata.Cathy.URN, nil, "foo", nil, nil, nil, flows.NilMsgTopic, envs.NilLocale, flows.NilUnsendableReason)
+	msg2 := flows.NewMsgOut(testdata.Cathy.URN, nil, "bar", nil, nil, nil, flows.NilMsgTopic, envs.NilLocale, flows.NilUnsendableReason)
 
 	assertRepetitions := func(m *flows.MsgOut, expected int) {
 		count, err := models.GetMsgRepetitions(rp, cathy, m)
@@ -694,7 +695,7 @@ func TestNewOutgoingIVR(t *testing.T) {
 
 	createdOn := time.Date(2021, 7, 26, 12, 6, 30, 0, time.UTC)
 
-	flowMsg := flows.NewIVRMsgOut(testdata.Cathy.URN, vonage.ChannelReference(), "Hello", "eng", "http://example.com/hi.mp3")
+	flowMsg := flows.NewIVRMsgOut(testdata.Cathy.URN, vonage.ChannelReference(), "Hello", "http://example.com/hi.mp3", "eng")
 	dbMsg := models.NewOutgoingIVR(rt.Config, testdata.Org1.ID, conn, flowMsg, createdOn)
 
 	assert.Equal(t, flowMsg.UUID(), dbMsg.UUID())

core/models/templates.go

@@ -56,11 +56,12 @@ func (t *TemplateTranslation) UnmarshalJSON(data []byte) error { return json.Unm
 func (t *TemplateTranslation) MarshalJSON() ([]byte, error) { return json.Marshal(t.t) }
 
 func (t *TemplateTranslation) Channel() assets.ChannelReference { return t.t.Channel }
-func (t *TemplateTranslation) Language() envs.Language          { return t.t.Language }
-func (t *TemplateTranslation) Country() envs.Country            { return envs.Country(t.t.Country) }
-func (t *TemplateTranslation) Content() string                  { return t.t.Content }
-func (t *TemplateTranslation) Namespace() string                { return t.t.Namespace }
-func (t *TemplateTranslation) VariableCount() int               { return t.t.VariableCount }
+func (t *TemplateTranslation) Locale() envs.Locale {
+	return envs.NewLocale(t.t.Language, envs.Country(t.t.Country))
+}
+func (t *TemplateTranslation) Content() string    { return t.t.Content }
+func (t *TemplateTranslation) Namespace() string  { return t.t.Namespace }
+func (t *TemplateTranslation) VariableCount() int { return t.t.VariableCount }
 
 // loads the templates for the passed in org
 func loadTemplates(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Template, error) {

core/models/templates_test.go

@@ -27,16 +27,14 @@ func TestTemplates(t *testing.T) {
 
 	assert.Equal(t, 1, len(templates[0].Translations()))
 	tt := templates[0].Translations()[0]
-	assert.Equal(t, envs.Language("fra"), tt.Language())
-	assert.Equal(t, envs.NilCountry, tt.Country())
+	assert.Equal(t, envs.Locale("fra"), tt.Locale())
 	assert.Equal(t, "", tt.Namespace())
 	assert.Equal(t, testdata.TwitterChannel.UUID, tt.Channel().UUID)
 	assert.Equal(t, "Salut!", tt.Content())
 
 	assert.Equal(t, 1, len(templates[1].Translations()))
 	tt = templates[1].Translations()[0]
-	assert.Equal(t, envs.Language("eng"), tt.Language())
-	assert.Equal(t, envs.Country("US"), tt.Country())
+	assert.Equal(t, envs.Locale("eng-US"), tt.Locale())
 	assert.Equal(t, "2d40b45c_25cd_4965_9019_f05d0124c5fa", tt.Namespace())
 	assert.Equal(t, testdata.TwitterChannel.UUID, tt.Channel().UUID)
 	assert.Equal(t, "Hi {{1}}, are you still experiencing problems with {{2}}?", tt.Content())

core/msgio/courier.go

@@ -180,8 +180,11 @@ func FetchAttachment(ctx context.Context, rt *runtime.Runtime, ch *models.Channe
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", rt.Config.CourierAuthToken))
 
 	resp, err := httpx.DoTrace(courierHttpClient, req, nil, nil, -1)
-	if err != nil || resp.Response.StatusCode != 200 {
-		return "", "", errors.New("error calling courier endpoint")
+	if err != nil {
+		return "", "", errors.Wrap(err, "error calling courier endpoint")
+	}
+	if resp.Response.StatusCode != 200 {
+		return "", "", errors.Errorf("error calling courier endpoint, got non-200 status: %s", string(resp.ResponseTrace))
 	}
 	fa := &fetchAttachmentResponse{}
 	if err := json.Unmarshal(resp.ResponseBody, fa); err != nil {

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/lib/pq v1.10.7
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.33.1
-	github.com/nyaruka/goflow v0.175.0
+	github.com/nyaruka/goflow v0.178.1
 	github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d
 	github.com/nyaruka/null v1.2.0
 	github.com/nyaruka/redisx v0.2.2

go.sum

@@ -221,8 +221,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
 github.com/nyaruka/gocommon v1.33.1 h1:RUy1O5Ly4tAaQDDpahds8z+4uewwsXg6SNCH0hYm7pE=
 github.com/nyaruka/gocommon v1.33.1/go.mod h1:gusIA2aNC8EPB3ozlP4O0PaBiHUNq5+f1peRNvcn0DI=
-github.com/nyaruka/goflow v0.175.0 h1:ofrTm5qlf19oR1mjg8wFCmvNS9faFyDIQiFNs039kss=
-github.com/nyaruka/goflow v0.175.0/go.mod h1:C3Hj+jvJ2RY6w/ANx4zjcbVjYzd8gzOcryyPW2OEa8E=
+github.com/nyaruka/goflow v0.178.1 h1:ubVQXcrlFIebDnfJOvDRMaGc3CyGpngrtJLiVDgsHDc=
+github.com/nyaruka/goflow v0.178.1/go.mod h1:C3Hj+jvJ2RY6w/ANx4zjcbVjYzd8gzOcryyPW2OEa8E=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc=

services/ivr/twiml/service.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/gocommon/urns"
-	"github.com/nyaruka/goflow/envs"
 	"github.com/nyaruka/goflow/flows"
 	"github.com/nyaruka/goflow/flows/events"
 	"github.com/nyaruka/goflow/flows/routers/waits/hints"
@@ -461,11 +460,8 @@ func ResponseForSprint(cfg *runtime.Config, urn urns.URN, resumeURL string, es [
 		switch event := e.(type) {
 		case *events.IVRCreatedEvent:
 			if len(event.Msg.Attachments()) == 0 {
-				urnCountry := envs.DeriveCountryFromTel(urn.Path())
-				msgLocale := envs.NewLocale(event.Msg.TextLanguage, urnCountry)
-
 				// only send locale if it's a supported say language for Twilio
-				msgLocaleCode := msgLocale.ToBCP47()
+				msgLocaleCode := event.Msg.Locale().ToBCP47()
 				if _, valid := supportedSayLanguages[msgLocaleCode]; !valid {
 					msgLocaleCode = ""
 				}

services/ivr/twiml/service_test.go

@@ -48,21 +48,21 @@ func TestResponseForSprint(t *testing.T) {
 		{
 			// ivr msg, supported text language specified
 			events: []flows.Event{
-				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Hi there", "eng", "")),
+				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Hi there", "", "eng-US")),
 			},
 			expected: `<Response><Say language="en-US">Hi there</Say><Hangup></Hangup></Response>`,
 		},
 		{
 			// ivr msg, unsupported text language specified
 			events: []flows.Event{
-				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Amakuru", "kin", "")),
+				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Amakuru", "", "kin")),
 			},
 			expected: `<Response><Say>Amakuru</Say><Hangup></Hangup></Response>`,
 		},
 		{
 			// ivr msg with audio attachment, text language ignored
 			events: []flows.Event{
-				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Hi there", "eng", "/recordings/foo.wav")),
+				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "Hi there", "/recordings/foo.wav", "eng-US")),
 			},
 			expected: `<Response><Play>https://mailroom.io/recordings/foo.wav</Play><Hangup></Hangup></Response>`,
 		},

services/ivr/vonage/service_test.go

@@ -81,13 +81,13 @@ func TestResponseForSprint(t *testing.T) {
 		},
 		{
 			[]flows.Event{
-				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "hello world", "", "/recordings/foo.wav")),
+				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "hello world", "/recordings/foo.wav", "")),
 			},
 			`[{"action":"stream","streamUrl":["/recordings/foo.wav"]}]`,
 		},
 		{
 			[]flows.Event{
-				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "hello world", "", "https://temba.io/recordings/foo.wav")),
+				events.NewIVRCreated(flows.NewIVRMsgOut(urn, channelRef, "hello world", "https://temba.io/recordings/foo.wav", "")),
 			},
 			`[{"action":"stream","streamUrl":["https://temba.io/recordings/foo.wav"]}]`,
 		},

testsuite/testdata/flows.go

@@ -34,8 +34,8 @@ func InsertFlow(db *sqlx.DB, org *Org, definition []byte) *Flow {
 
 	var id models.FlowID
 	must(db.Get(&id,
-		`INSERT INTO flows_flow(org_id, uuid, name, flow_type, version_number, expires_after_minutes, ignore_triggers, has_issues, is_active, is_archived, is_system, created_by_id, created_on, modified_by_id, modified_on, saved_on, saved_by_id) 
-		VALUES($1, $2, $3, 'M', 1, 10, FALSE, FALSE, TRUE, FALSE, FALSE, $4, NOW(), $4, NOW(), NOW(), $4) RETURNING id`, org.ID, uuid, name, Admin.ID,
+		`INSERT INTO flows_flow(org_id, uuid, name, flow_type, version_number, base_language, expires_after_minutes, ignore_triggers, has_issues, is_active, is_archived, is_system, created_by_id, created_on, modified_by_id, modified_on, saved_on, saved_by_id) 
+		VALUES($1, $2, $3, 'M', '13.1.0', 'eng', 10, FALSE, FALSE, TRUE, FALSE, FALSE, $4, NOW(), $4, NOW(), NOW(), $4) RETURNING id`, org.ID, uuid, name, Admin.ID,
 	))
 
 	db.MustExec(`INSERT INTO flows_flowrevision(flow_id, definition, spec_version, revision, is_active, created_by_id, created_on, modified_by_id, modified_on) 

testsuite/testdata/msgs.go

@@ -54,7 +54,7 @@ func insertOutgoingMsg(db *sqlx.DB, org *Org, channel *Channel, contact *Contact
 		channelID = channel.ID
 	}
 
-	msg := flows.NewMsgOut(contact.URN, channelRef, text, attachments, nil, nil, flows.NilMsgTopic, flows.NilUnsendableReason)
+	msg := flows.NewMsgOut(contact.URN, channelRef, text, attachments, nil, nil, flows.NilMsgTopic, envs.NilLocale, flows.NilUnsendableReason)
 
 	var sentOn *time.Time
 	if status == models.MsgStatusWired || status == models.MsgStatusSent || status == models.MsgStatusDelivered {

testsuite/testsuite.go

@@ -197,7 +197,7 @@ DELETE FROM msgs_msg;
 DELETE FROM flows_flowrun;
 DELETE FROM flows_flowpathcount;
 DELETE FROM flows_flownodecount;
-DELETE FROM flows_flowruncount;
+DELETE FROM flows_flowrunstatuscount;
 DELETE FROM flows_flowcategorycount;
 DELETE FROM flows_flowsession;
 DELETE FROM flows_flowrevision WHERE flow_id >= 30000;

web/ivr/ivr_test.go

@@ -199,7 +199,7 @@ func TestTwilioIVR(t *testing.T) {
 				`<Say>You said</Say>`,
 				`<Say>I hope hearing that makes you feel better. Good day and good bye.</Say>`,
 				`<Dial action=`,
-				`>2065551212</Dial>`,
+				`>+12065551212</Dial>`,
 			},
 			expectedConnStatus: map[string]string{"Call1": "I", "Call2": "W", "Call3": "W"},
 		},
@@ -364,7 +364,7 @@ func mockVonageHandler(w http.ResponseWriter, r *http.Request) {
 		} else if form.To[0].Number == "16055743333" {
 			w.WriteHeader(http.StatusCreated)
 			w.Write([]byte(`{ "uuid": "Call2","status": "started","direction": "outbound","conversation_uuid": "Conversation2"}`))
-		} else if form.To[0].Number == "2065551212" {
+		} else if form.To[0].Number == "12065551212" {
 			// start of a transfer leg
 			w.WriteHeader(http.StatusCreated)
 			w.Write([]byte(`{ "uuid": "Call3","status": "started","direction": "outbound","conversation_uuid": "Conversation3"}`))