diff options
Diffstat (limited to 'core/operator')
| -rw-r--r-- | core/operator/harness_test.go | 23 | ||||
| -rw-r--r-- | core/operator/watermark.go | 10 | ||||
| -rw-r--r-- | core/operator/watermark_test.go | 58 |
3 files changed, 83 insertions, 8 deletions
diff --git a/core/operator/harness_test.go b/core/operator/harness_test.go new file mode 100644 index 0000000..383ab8c --- /dev/null +++ b/core/operator/harness_test.go | |||
| @@ -0,0 +1,23 @@ | |||
| 1 | package operator_test | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | |||
| 6 | "flux/core" | ||
| 7 | ) | ||
| 8 | |||
| 9 | type captureEmitter[T any] struct { | ||
| 10 | records []core.StreamRecord[T] | ||
| 11 | } | ||
| 12 | |||
| 13 | func (e *captureEmitter[T]) Emit(_ context.Context, record core.StreamRecord[T]) error { | ||
| 14 | e.records = append(e.records, record) | ||
| 15 | return nil | ||
| 16 | } | ||
| 17 | |||
| 18 | func (e *captureEmitter[T]) Records() []core.StreamRecord[T] { | ||
| 19 | result := make([]core.StreamRecord[T], len(e.records)) | ||
| 20 | copy(result, e.records) | ||
| 21 | |||
| 22 | return result | ||
| 23 | } | ||
diff --git a/core/operator/watermark.go b/core/operator/watermark.go index 3d591e5..43d63fa 100644 --- a/core/operator/watermark.go +++ b/core/operator/watermark.go | |||
| @@ -6,20 +6,15 @@ import ( | |||
| 6 | "flux/core" | 6 | "flux/core" |
| 7 | ) | 7 | ) |
| 8 | 8 | ||
| 9 | // TimestampAssigner is responsible for event time defined by user | ||
| 10 | type TimestampAssigner[IN any] func(in IN) int64 | ||
| 11 | |||
| 12 | type WaterGeneratorOperator[IN any] struct { | 9 | type WaterGeneratorOperator[IN any] struct { |
| 13 | assigner TimestampAssigner[IN] | ||
| 14 | maxOutOfOrder int64 | 10 | maxOutOfOrder int64 |
| 15 | currentMaxTimestamp int64 | 11 | currentMaxTimestamp int64 |
| 16 | lastEmittedWatermark int64 | 12 | lastEmittedWatermark int64 |
| 17 | } | 13 | } |
| 18 | 14 | ||
| 19 | func NewWatermarkGeneratorOperator[IN any](maxOutOfOrder int64, assigner TimestampAssigner[IN]) *WaterGeneratorOperator[IN] { | 15 | func NewWatermarkGeneratorOperator[IN any](maxOutOfOrder int64) *WaterGeneratorOperator[IN] { |
| 20 | return &WaterGeneratorOperator[IN]{ | 16 | return &WaterGeneratorOperator[IN]{ |
| 21 | maxOutOfOrder: maxOutOfOrder, | 17 | maxOutOfOrder: maxOutOfOrder, |
| 22 | assigner: assigner, | ||
| 23 | lastEmittedWatermark: -1, | 18 | lastEmittedWatermark: -1, |
| 24 | } | 19 | } |
| 25 | } | 20 | } |
| @@ -34,8 +29,7 @@ func (w *WaterGeneratorOperator[IN]) Process(ctx context.Context, record core.St | |||
| 34 | return err | 29 | return err |
| 35 | } | 30 | } |
| 36 | // query the time | 31 | // query the time |
| 37 | eventTime := w.assigner(record.Value) | 32 | eventTime := record.Timestamp |
| 38 | record.Timestamp = eventTime | ||
| 39 | 33 | ||
| 40 | // update the max timestamp | 34 | // update the max timestamp |
| 41 | if w.currentMaxTimestamp < eventTime { | 35 | if w.currentMaxTimestamp < eventTime { |
diff --git a/core/operator/watermark_test.go b/core/operator/watermark_test.go new file mode 100644 index 0000000..0a0495e --- /dev/null +++ b/core/operator/watermark_test.go | |||
| @@ -0,0 +1,58 @@ | |||
| 1 | package operator_test | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | "flux/core/operator" | ||
| 7 | "testing" | ||
| 8 | ) | ||
| 9 | |||
| 10 | type timestampEvent struct { | ||
| 11 | EventTime int64 | ||
| 12 | } | ||
| 13 | |||
| 14 | func TestWatermarkGeneratorPreservesAssignedEventTimestamp(t *testing.T) { | ||
| 15 | ctx := context.Background() | ||
| 16 | |||
| 17 | op := operator.NewWatermarkGeneratorOperator[timestampEvent](0) | ||
| 18 | |||
| 19 | if err := op.Open(ctx); err != nil { | ||
| 20 | t.Fatalf("open operator failed: %v", err) | ||
| 21 | } | ||
| 22 | |||
| 23 | t.Cleanup(func() { | ||
| 24 | if err := op.Close(ctx); err != nil { | ||
| 25 | t.Errorf("close operator: %v", err) | ||
| 26 | } | ||
| 27 | }) | ||
| 28 | |||
| 29 | out := &captureEmitter[timestampEvent]{} | ||
| 30 | input := core.NewDataRecord("user-1", timestampEvent{EventTime: 999}, 40) | ||
| 31 | |||
| 32 | if err := op.Process(ctx, input, out); err != nil { | ||
| 33 | t.Fatalf("process operator failed: %v", err) | ||
| 34 | } | ||
| 35 | |||
| 36 | records := out.Records() | ||
| 37 | if len(records) == 0 { | ||
| 38 | t.Fatal("operator emitted no records") | ||
| 39 | } | ||
| 40 | |||
| 41 | got := records[0] | ||
| 42 | |||
| 43 | if got.Type != core.RecordData { | ||
| 44 | t.Fatalf("first output type: got %v, want RecordData", got.Type) | ||
| 45 | } | ||
| 46 | |||
| 47 | if got.Key != "user-1" { | ||
| 48 | t.Errorf("data key: got %q, want %q", got.Key, "user-1") | ||
| 49 | } | ||
| 50 | |||
| 51 | if got.Value != input.Value { | ||
| 52 | t.Errorf("data value: got %+v, want %+v", got.Value, input.Value) | ||
| 53 | } | ||
| 54 | |||
| 55 | if got.Timestamp != 40 { | ||
| 56 | t.Errorf("data timestamp: got %d, want %d", got.Timestamp, 40) | ||
| 57 | } | ||
| 58 | } | ||
