fix(ddd): fix mysql diff with json (#24)

* feat(tool): add tool config

* fix(ddd): fix mysql diff with json

* fix(ddd): fix mysql diff with json

* fix(stage): fix get *IEntity

* fix(engine): fix entiry query

* fix(ddd): fix entiry query

* rebase: master

* fix(diff): fix update delete random order

---------

Co-authored-by: lujinghao <lujinghao@bytedance.com>

.gitignore

@@ -10,3 +10,4 @@ mysql-data
 *.sqlite3
 .vscode
 **/*.DS_Store
+vendor

engine.go

@@ -649,6 +649,7 @@ func (e *Stage) BuildEntity(ctx context.Context, parent IEntity, children ...int
 
 	for _, item := range children {
 		itemType := reflect.TypeOf(item)
+		itemVal := reflect.ValueOf(item)
 		if itemType.Kind() != reflect.Ptr {
 			return fmt.Errorf("children must be pointer")
 		}
@@ -656,6 +657,20 @@ func (e *Stage) BuildEntity(ctx context.Context, parent IEntity, children ...int
 			if err := e.buildEntitySliceByParent(ctx, parent, item); err != nil {
 				return err
 			}
+		} else if itemType.Elem().Kind() == reflect.Ptr &&
+			itemType.Elem().Implements(entityType) {
+			if itemVal.Elem().IsNil() && itemVal.Elem().CanSet() {
+				newItem := reflect.New(itemType.Elem().Elem())
+				itemVal.Elem().Set(newItem)
+			}
+			itemDef := itemVal.Elem().Interface()
+			if err := e.buildEntity(ctx, itemDef.(IEntity), parent); err != nil {
+				if errors.Is(err, ErrEntityNotFound) {
+					itemVal.Elem().SetZero()
+				} else {
+					return err
+				}
+			}
 		} else if itemType.Implements(entityType) {
 			if err := e.buildEntity(ctx, item.(IEntity), parent); err != nil && !errors.Is(err, ErrEntityNotFound) {
 				return err
@@ -670,9 +685,9 @@ func (e *Stage) BuildEntity(ctx context.Context, parent IEntity, children ...int
 // 查询并构建 entity 实体，注意，不会处理 parent 实体
 func (e *Stage) buildEntity(ctx context.Context, entity, parent IEntity) error {
 	// 至少一个有 ID
-	if entity.GetID() == "" && parent.GetID() == "" {
-		return fmt.Errorf("entity to build must has id")
-	}
+	// if entity.GetID() == "" && parent.GetID() == "" {
+	// 	return fmt.Errorf("entity to build must has id")
+	// }
 	po, err := e.executor.Entity2Model(entity, parent, OpQuery)
 	if err != nil {
 		return err

executor/mysql/diff.go

@@ -33,28 +33,58 @@ func hasValue(tag, attr string) bool {
 	return false
 }
 
+func diffValue(a, b reflect.Value) (fields []string, diff bool) {
+	a, b = reflect.Indirect(a), reflect.Indirect(b)
+	if !a.IsValid() || !b.IsValid() {
+		if a.IsValid() {
+			b = reflect.New(a.Type()).Elem()
+		} else if b.IsValid() {
+			a = reflect.New(b.Type()).Elem()
+		} else {
+			diff = false
+			return
+		}
+	}
+	if a.IsZero() && b.IsZero() {
+		diff = false
+		return
+	}
+	switch {
+	case a.Kind() == reflect.Struct:
+		fields = diffStruct(a, b)
+		diff = len(fields) > 0
+	case a.Comparable() && b.Comparable():
+		diff = !a.Equal(b)
+	case a.CanInterface() && b.CanInterface():
+		diff = !reflect.DeepEqual(a.Interface(), b.Interface())
+	default:
+		diff = true
+	}
+	return
+}
+
 func diffStruct(currVal, prevVal reflect.Value) []string {
 	result := make([]string, 0)
 	poType := currVal.Type()
 	for i := 0; i < currVal.NumField(); i++ {
 		field := poType.Field(i)
-		fieldVal, prevFiledVal := currVal.Field(i), prevVal.Field(i)
-		if !fieldVal.CanInterface() {
-			continue
-		}
+		fieldVal := currVal.Field(i)
+		prevFiledVal := prevVal.Field(i)
+		fieldVal, prevFiledVal = reflect.Indirect(fieldVal), reflect.Indirect(prevFiledVal)
 		fieldName := field.Name
 		fieldTag := field.Tag.Get("gorm")
-		if fieldVal.Kind() == reflect.Struct && (field.Anonymous || hasValue(fieldTag, "embedded")) {
-			structDiff := diffStruct(fieldVal, prevFiledVal)
-			result = append(result, structDiff...)
-		} else if fieldVal.Comparable() {
-			if fieldVal.Equal(prevFiledVal) {
-				continue
+		if fieldVal.Kind() == reflect.Struct {
+			if structDiff, diff := diffValue(fieldVal, prevFiledVal); diff {
+				if field.Anonymous || hasValue(fieldTag, "embedded") {
+					result = append(result, structDiff...)
+				} else {
+					result = append(result, fieldName)
+				}
 			}
-			result = append(result, fieldName)
 		} else {
-			// todo 不能比对的字段先一律更新，待优化比对方案
-			result = append(result, fieldName)
+			if _, diff := diffValue(fieldVal, prevFiledVal); diff {
+				result = append(result, fieldName)
+			}
 		}
 	}
 	return result

executor/mysql/diff_test.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"gorm.io/datatypes"
 )
 
 type TestDiffBasePO struct {
@@ -27,15 +28,28 @@ type TestDiffBasePO struct {
 	UpdatedAt time.Time
 }
 
+type NestStruct struct {
+	NestName string
+	NestNum  int64
+}
+
 type testDiffPO struct {
 	TestDiffBasePO `gorm:"embedded;embeddedPrefix:base_"`
-
-	ID           string
-	Name         string
-	ItemPrice    int
-	DiffPtrValue *int
-	SamePtrValue *int
-	SliceValue   []string
+	NestStruct
+	ID              string
+	Name            string
+	ItemPrice       int
+	DiffPtrValue    *int
+	SamePtrValue    *int
+	SliceValue      []string
+	SliceStruct     []*NestStruct
+	JSONValue       datatypes.JSON
+	SameJSONValue   datatypes.JSON
+	StructValue     *NestStruct `gorm:"type:json,serialize:json"`
+	SameStructValue *NestStruct `gorm:"type:json,serialize:json"`
+	EmptyStruct1    *NestStruct `gorm:"type:json,serialize:json"`
+	EmptyStruct2    *NestStruct `gorm:"type:json,serialize:json"`
+	EmptyStruct3    *NestStruct `gorm:"type:json,serialize:json"`
 }
 
 func TestDiffModel(t *testing.T) {
@@ -46,12 +60,32 @@ func TestDiffModel(t *testing.T) {
 			CreatedAt: time.Now(),
 			UpdatedAt: now,
 		},
-		ID:           "p1",
-		Name:         "n2",
-		ItemPrice:    100,
-		DiffPtrValue: &i,
-		SamePtrValue: &i,
-		SliceValue:   []string{"abc"},
+		NestStruct: NestStruct{
+			NestNum: 100,
+		},
+		SliceStruct: []*NestStruct{
+			{NestName: "test"},
+		},
+		ID:            "p1",
+		Name:          "n2",
+		ItemPrice:     100,
+		DiffPtrValue:  &i,
+		SamePtrValue:  &i,
+		SliceValue:    []string{"abc"},
+		JSONValue:     datatypes.JSON([]byte(`{"a":1}`)),
+		SameJSONValue: datatypes.JSON([]byte(`{"a":1}`)),
+		StructValue: &NestStruct{
+			NestName: "s1",
+			NestNum:  1,
+		},
+		SameStructValue: &NestStruct{
+			NestName: "s1",
+			NestNum:  2,
+		},
+		EmptyStruct1: &NestStruct{
+			NestName: "s1",
+			NestNum:  3,
+		},
 	}
 
 	p2 := testDiffPO{
@@ -64,14 +98,25 @@ func TestDiffModel(t *testing.T) {
 		DiffPtrValue: &j,
 		SamePtrValue: &i,
 		SliceValue:   []string{"kkk"},
+		SliceStruct: []*NestStruct{
+			{NestName: "test2"},
+		},
+		JSONValue:     datatypes.JSON([]byte(`{"a":2}`)),
+		SameJSONValue: datatypes.JSON([]byte(`{"a":1}`)),
+		StructValue: &NestStruct{
+			NestName: "s2",
+			NestNum:  1,
+		},
+		SameStructValue: &NestStruct{
+			NestName: "s1",
+			NestNum:  2,
+		},
+		EmptyStruct2: &NestStruct{
+			NestName: "s1",
+			NestNum:  3,
+		},
 	}
 
 	result := DiffModel(p1, p2)
-	assert.NotEmpty(t, result)
-	assert.Contains(t, result, "ID")
-	assert.Contains(t, result, "ItemPrice")
-	assert.Contains(t, result, "DiffPtrValue")
-	assert.Contains(t, result, "CreatedAt")
-	assert.Contains(t, result, "SliceValue")
-	assert.NotContains(t, result, "SamePtrValue")
+	assert.ElementsMatch(t, result, []string{"ID", "ItemPrice", "DiffPtrValue", "CreatedAt", "SliceValue", "JSONValue", "StructValue", "UpdatedAt", "EmptyStruct1", "EmptyStruct2", "NestNum", "SliceStruct"})
 }

executor/mysql/executor.go

@@ -88,6 +88,9 @@ var opMap = map[ddd.OpType]execFunc{
 		for i, m := range a.Models {
 			if len(a.PrevModels) > i {
 				fields := DiffModel(m, a.PrevModels[i])
+				if len(fields) == 0 {
+					continue
+				}
 				if err := db.Select(fields).Updates(m).Error; err != nil {
 					return err
 				}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/robfig/cron v1.2.0
 	github.com/rs/xid v1.5.0
 	github.com/stretchr/testify v1.8.3
+	gorm.io/datatypes v1.2.0
 	gorm.io/driver/mysql v1.5.1
 	gorm.io/driver/sqlite v1.5.2
 	gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55

go.sum

@@ -17,6 +17,11 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -28,6 +33,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
@@ -38,15 +44,21 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco=
+gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04=
 gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
 gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
+gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U=
 gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc=
 gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
+gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0=
 gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ=
 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

snapshot.go

@@ -208,7 +208,14 @@ func walk(entity, parent IEntity, f func(entity, parent IEntity, children map[st
 // entityDiff 递归对比实体所有层级的子实体，生成子实体的差异
 func entityDiff(parent IEntity, pool entitySnapshotPool) []*entityChanged {
 	res := make([]*entityChanged, 0)
-	for key, children := range findChildren(parent) {
+	childrenList := findChildren(parent)
+	keys := []string{}
+	for key := range childrenList {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		children := childrenList[key]
 		s1 := makeEntitySet(children)
 		s2 := simpleSet{}
 		if pool[parent] != nil && len(pool[parent].children[key]) > 0 {