aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
Diffstat (limited to 'core')
-rw-r--r--core/operator/harness_test.go23
-rw-r--r--core/operator/watermark.go10
-rw-r--r--core/operator/watermark_test.go58
3 files changed, 83 insertions, 8 deletions
diff --git a/core/operator/harness_test.go b/core/operator/harness_test.go
new file mode 100644
index 0000000..383ab8c
--- /dev/null
+++ b/core/operator/harness_test.go
@@ -0,0 +1,23 @@
1package operator_test
2
3import (
4 "context"
5
6 "flux/core"
7)
8
9type captureEmitter[T any] struct {
10 records []core.StreamRecord[T]
11}
12
13func (e *captureEmitter[T]) Emit(_ context.Context, record core.StreamRecord[T]) error {
14 e.records = append(e.records, record)
15 return nil
16}
17
18func (e *captureEmitter[T]) Records() []core.StreamRecord[T] {
19 result := make([]core.StreamRecord[T], len(e.records))
20 copy(result, e.records)
21
22 return result
23}
diff --git a/core/operator/watermark.go b/core/operator/watermark.go
index 3d591e5..43d63fa 100644
--- a/core/operator/watermark.go
+++ b/core/operator/watermark.go
@@ -6,20 +6,15 @@ import (
6 "flux/core" 6 "flux/core"
7) 7)
8 8
9// TimestampAssigner is responsible for event time defined by user
10type TimestampAssigner[IN any] func(in IN) int64
11
12type WaterGeneratorOperator[IN any] struct { 9type WaterGeneratorOperator[IN any] struct {
13 assigner TimestampAssigner[IN]
14 maxOutOfOrder int64 10 maxOutOfOrder int64
15 currentMaxTimestamp int64 11 currentMaxTimestamp int64
16 lastEmittedWatermark int64 12 lastEmittedWatermark int64
17} 13}
18 14
19func NewWatermarkGeneratorOperator[IN any](maxOutOfOrder int64, assigner TimestampAssigner[IN]) *WaterGeneratorOperator[IN] { 15func NewWatermarkGeneratorOperator[IN any](maxOutOfOrder int64) *WaterGeneratorOperator[IN] {
20 return &WaterGeneratorOperator[IN]{ 16 return &WaterGeneratorOperator[IN]{
21 maxOutOfOrder: maxOutOfOrder, 17 maxOutOfOrder: maxOutOfOrder,
22 assigner: assigner,
23 lastEmittedWatermark: -1, 18 lastEmittedWatermark: -1,
24 } 19 }
25} 20}
@@ -34,8 +29,7 @@ func (w *WaterGeneratorOperator[IN]) Process(ctx context.Context, record core.St
34 return err 29 return err
35 } 30 }
36 // query the time 31 // query the time
37 eventTime := w.assigner(record.Value) 32 eventTime := record.Timestamp
38 record.Timestamp = eventTime
39 33
40 // update the max timestamp 34 // update the max timestamp
41 if w.currentMaxTimestamp < eventTime { 35 if w.currentMaxTimestamp < eventTime {
diff --git a/core/operator/watermark_test.go b/core/operator/watermark_test.go
new file mode 100644
index 0000000..0a0495e
--- /dev/null
+++ b/core/operator/watermark_test.go
@@ -0,0 +1,58 @@
1package operator_test
2
3import (
4 "context"
5 "flux/core"
6 "flux/core/operator"
7 "testing"
8)
9
10type timestampEvent struct {
11 EventTime int64
12}
13
14func TestWatermarkGeneratorPreservesAssignedEventTimestamp(t *testing.T) {
15 ctx := context.Background()
16
17 op := operator.NewWatermarkGeneratorOperator[timestampEvent](0)
18
19 if err := op.Open(ctx); err != nil {
20 t.Fatalf("open operator failed: %v", err)
21 }
22
23 t.Cleanup(func() {
24 if err := op.Close(ctx); err != nil {
25 t.Errorf("close operator: %v", err)
26 }
27 })
28
29 out := &captureEmitter[timestampEvent]{}
30 input := core.NewDataRecord("user-1", timestampEvent{EventTime: 999}, 40)
31
32 if err := op.Process(ctx, input, out); err != nil {
33 t.Fatalf("process operator failed: %v", err)
34 }
35
36 records := out.Records()
37 if len(records) == 0 {
38 t.Fatal("operator emitted no records")
39 }
40
41 got := records[0]
42
43 if got.Type != core.RecordData {
44 t.Fatalf("first output type: got %v, want RecordData", got.Type)
45 }
46
47 if got.Key != "user-1" {
48 t.Errorf("data key: got %q, want %q", got.Key, "user-1")
49 }
50
51 if got.Value != input.Value {
52 t.Errorf("data value: got %+v, want %+v", got.Value, input.Value)
53 }
54
55 if got.Timestamp != 40 {
56 t.Errorf("data timestamp: got %d, want %d", got.Timestamp, 40)
57 }
58}