diff options
Diffstat (limited to 'core')
| -rw-r--r-- | core/hash.go | 19 | ||||
| -rw-r--r-- | core/operator/filter.go | 36 | ||||
| -rw-r--r-- | core/operator/keyby.go | 32 | ||||
| -rw-r--r-- | core/operator/keyed_process.go | 81 | ||||
| -rw-r--r-- | core/operator/map.go | 53 | ||||
| -rw-r--r-- | core/operator/operator.go | 16 | ||||
| -rw-r--r-- | core/operator/stateful_map.go | 75 | ||||
| -rw-r--r-- | core/operator/watermark.go | 61 | ||||
| -rw-r--r-- | core/operator/window.go | 110 | ||||
| -rw-r--r-- | core/record.go | 40 | ||||
| -rw-r--r-- | core/state.go | 10 | ||||
| -rw-r--r-- | core/state_backend.go | 62 | ||||
| -rw-r--r-- | core/timer.go | 113 | ||||
| -rw-r--r-- | core/window.go | 54 |
14 files changed, 762 insertions, 0 deletions
diff --git a/core/hash.go b/core/hash.go new file mode 100644 index 0000000..af15546 --- /dev/null +++ b/core/hash.go | |||
| @@ -0,0 +1,19 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | import "hash/fnv" | ||
| 4 | |||
| 5 | const DefaultMaxParallelism = 128 | ||
| 6 | |||
| 7 | func AssignToKeyGroup(key string, maxParallelism int) int { | ||
| 8 | |||
| 9 | if maxParallelism <= 0 { | ||
| 10 | maxParallelism = DefaultMaxParallelism | ||
| 11 | } | ||
| 12 | |||
| 13 | // hash | ||
| 14 | h := fnv.New32a() | ||
| 15 | h.Write([]byte(key)) | ||
| 16 | hash := h.Sum32() | ||
| 17 | |||
| 18 | return int(hash % uint32(maxParallelism)) | ||
| 19 | } | ||
diff --git a/core/operator/filter.go b/core/operator/filter.go new file mode 100644 index 0000000..80bbb77 --- /dev/null +++ b/core/operator/filter.go | |||
| @@ -0,0 +1,36 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | ) | ||
| 7 | |||
| 8 | type FilterFunc[IN any] func(in IN) bool | ||
| 9 | |||
| 10 | type FilterOperator[IN any] struct { | ||
| 11 | userFunc FilterFunc[IN] | ||
| 12 | } | ||
| 13 | |||
| 14 | func NewFilterOperator[IN any](userFunc FilterFunc[IN]) *FilterOperator[IN] { | ||
| 15 | return &FilterOperator[IN]{userFunc: userFunc} | ||
| 16 | } | ||
| 17 | |||
| 18 | func (f *FilterOperator[IN]) Open(ctx context.Context) error { | ||
| 19 | return nil | ||
| 20 | } | ||
| 21 | |||
| 22 | func (f *FilterOperator[IN]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[IN]) error { | ||
| 23 | if record.Type != core.RecordData { | ||
| 24 | return out.Emit(ctx, record) | ||
| 25 | } | ||
| 26 | |||
| 27 | if f.userFunc(record.Value) { | ||
| 28 | return out.Emit(ctx, record) | ||
| 29 | } | ||
| 30 | |||
| 31 | return nil | ||
| 32 | } | ||
| 33 | |||
| 34 | func (f *FilterOperator[IN]) Close(ctx context.Context) error { | ||
| 35 | return nil | ||
| 36 | } | ||
diff --git a/core/operator/keyby.go b/core/operator/keyby.go new file mode 100644 index 0000000..56ef34e --- /dev/null +++ b/core/operator/keyby.go | |||
| @@ -0,0 +1,32 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | ) | ||
| 7 | |||
| 8 | type KeySelector[IN any] func(in IN) string | ||
| 9 | type KeyByOperator[IN any] struct { | ||
| 10 | selector KeySelector[IN] | ||
| 11 | } | ||
| 12 | |||
| 13 | func NewKeyByOperator[IN any](selector KeySelector[IN]) *KeyByOperator[IN] { | ||
| 14 | return &KeyByOperator[IN]{selector: selector} | ||
| 15 | } | ||
| 16 | |||
| 17 | func (k *KeyByOperator[IN]) Open(ctx context.Context) error { | ||
| 18 | return nil | ||
| 19 | } | ||
| 20 | |||
| 21 | func (k *KeyByOperator[IN]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[IN]) error { | ||
| 22 | if record.Type != core.RecordData { | ||
| 23 | return out.Emit(ctx, record) | ||
| 24 | } | ||
| 25 | |||
| 26 | record.Key = k.selector(record.Value) | ||
| 27 | return out.Emit(ctx, record) | ||
| 28 | } | ||
| 29 | |||
| 30 | func (k *KeyByOperator[IN]) Close(ctx context.Context) error { | ||
| 31 | return nil | ||
| 32 | } | ||
diff --git a/core/operator/keyed_process.go b/core/operator/keyed_process.go new file mode 100644 index 0000000..1d3c86c --- /dev/null +++ b/core/operator/keyed_process.go | |||
| @@ -0,0 +1,81 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | "fmt" | ||
| 7 | ) | ||
| 8 | |||
| 9 | type KeyedProcesser[IN, OUT any] interface { | ||
| 10 | Open(ctx context.Context) error | ||
| 11 | ProcessElement(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error | ||
| 12 | Close(ctx context.Context) error | ||
| 13 | OnTimer(ctx context.Context, timestamp int64, out Emitter[OUT]) error | ||
| 14 | } | ||
| 15 | |||
| 16 | type KeyProcessOperator[IN, OUT any] struct { | ||
| 17 | userFunc KeyedProcesser[IN, OUT] | ||
| 18 | timeService core.InternalTimerService | ||
| 19 | backend core.StateBackend | ||
| 20 | } | ||
| 21 | |||
| 22 | func NewKeyProcessOperator[IN, OUT any](userFunc KeyedProcesser[IN, OUT]) *KeyProcessOperator[IN, OUT] { | ||
| 23 | return &KeyProcessOperator[IN, OUT]{ | ||
| 24 | userFunc: userFunc, | ||
| 25 | timeService: core.NewTimerService(), | ||
| 26 | } | ||
| 27 | } | ||
| 28 | |||
| 29 | func (k *KeyProcessOperator[IN, OUT]) Open(ctx context.Context) error { | ||
| 30 | // init backend | ||
| 31 | be, ok := core.ExtractStateBackend(ctx) | ||
| 32 | if !ok { | ||
| 33 | return fmt.Errorf("state backend not found") | ||
| 34 | } | ||
| 35 | |||
| 36 | k.backend = be | ||
| 37 | // inject the timer to context for user use | ||
| 38 | ctx = core.InjectTimerService(ctx, k.timeService) | ||
| 39 | return k.userFunc.Open(ctx) | ||
| 40 | |||
| 41 | } | ||
| 42 | |||
| 43 | func (k *KeyProcessOperator[IN, OUT]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error { | ||
| 44 | |||
| 45 | switch record.Type { | ||
| 46 | case core.RecordData: | ||
| 47 | if err := k.backend.SetCurrentKey(record.Key); err != nil { | ||
| 48 | return err | ||
| 49 | } | ||
| 50 | |||
| 51 | return k.userFunc.ProcessElement(ctx, record, out) | ||
| 52 | case core.RecordWatermark: | ||
| 53 | // advance the timer | ||
| 54 | triggerTimerQueue := k.timeService.AdvanceWatermark(record.Timestamp) | ||
| 55 | |||
| 56 | for _, t := range triggerTimerQueue { | ||
| 57 | |||
| 58 | if err := k.backend.SetCurrentKey(t.Key); err != nil { | ||
| 59 | return err | ||
| 60 | } | ||
| 61 | if err := k.userFunc.OnTimer(ctx, t.Timestamp, out); err != nil { | ||
| 62 | return err | ||
| 63 | } | ||
| 64 | } | ||
| 65 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 66 | Type: core.RecordWatermark, | ||
| 67 | Timestamp: record.Timestamp, | ||
| 68 | }) | ||
| 69 | default: | ||
| 70 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 71 | Timestamp: record.Timestamp, | ||
| 72 | BarrierID: record.BarrierID, | ||
| 73 | Key: record.Key, | ||
| 74 | }) | ||
| 75 | } | ||
| 76 | |||
| 77 | } | ||
| 78 | |||
| 79 | func (k *KeyProcessOperator[IN, OUT]) Close(ctx context.Context) error { | ||
| 80 | return k.userFunc.Close(ctx) | ||
| 81 | } | ||
diff --git a/core/operator/map.go b/core/operator/map.go new file mode 100644 index 0000000..2165c25 --- /dev/null +++ b/core/operator/map.go | |||
| @@ -0,0 +1,53 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | "fmt" | ||
| 7 | ) | ||
| 8 | |||
| 9 | // MapFunc is business logic for user implement | ||
| 10 | type MapFunc[IN, OUT any] func(in IN) (OUT, error) | ||
| 11 | |||
| 12 | type MapOperator[IN, OUT any] struct { | ||
| 13 | userFunc MapFunc[IN, OUT] | ||
| 14 | } | ||
| 15 | |||
| 16 | func NewMapOperator[IN, OUT any](fn MapFunc[IN, OUT]) *MapOperator[IN, OUT] { | ||
| 17 | return &MapOperator[IN, OUT]{ | ||
| 18 | userFunc: fn, | ||
| 19 | } | ||
| 20 | } | ||
| 21 | |||
| 22 | func (fo *MapOperator[IN, OUT]) Open(ctx context.Context) error { | ||
| 23 | return nil | ||
| 24 | } | ||
| 25 | |||
| 26 | func (fo *MapOperator[IN, OUT]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error { | ||
| 27 | if record.Type != core.RecordData { | ||
| 28 | // watermark and barrier | ||
| 29 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 30 | Type: record.Type, | ||
| 31 | Key: record.Key, | ||
| 32 | Timestamp: record.Timestamp, | ||
| 33 | BarrierID: record.BarrierID, | ||
| 34 | }) | ||
| 35 | } | ||
| 36 | |||
| 37 | outValue, err := fo.userFunc(record.Value) | ||
| 38 | |||
| 39 | if err != nil { | ||
| 40 | return fmt.Errorf("cannot process value %v: %w", record.Value, err) | ||
| 41 | } | ||
| 42 | |||
| 43 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 44 | Type: core.RecordData, | ||
| 45 | Key: record.Key, | ||
| 46 | Timestamp: record.Timestamp, | ||
| 47 | Value: outValue, | ||
| 48 | }) | ||
| 49 | } | ||
| 50 | |||
| 51 | func (fo *MapOperator[IN, OUT]) Close(ctx context.Context) error { | ||
| 52 | return nil | ||
| 53 | } | ||
diff --git a/core/operator/operator.go b/core/operator/operator.go new file mode 100644 index 0000000..0c588e5 --- /dev/null +++ b/core/operator/operator.go | |||
| @@ -0,0 +1,16 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "flux/core" | ||
| 6 | ) | ||
| 7 | |||
| 8 | type Emitter[OUT any] interface { | ||
| 9 | Emit(ctx context.Context, record core.StreamRecord[OUT]) error | ||
| 10 | } | ||
| 11 | |||
| 12 | type Operator[IN, OUT any] interface { | ||
| 13 | Open(ctx context.Context) error | ||
| 14 | Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error | ||
| 15 | Close(ctx context.Context) error | ||
| 16 | } | ||
diff --git a/core/operator/stateful_map.go b/core/operator/stateful_map.go new file mode 100644 index 0000000..295c656 --- /dev/null +++ b/core/operator/stateful_map.go | |||
| @@ -0,0 +1,75 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "fmt" | ||
| 6 | |||
| 7 | "flux/core" | ||
| 8 | ) | ||
| 9 | |||
| 10 | // RichMapFunc is a stateful map on the user side: Open grabs the state handles | ||
| 11 | // off ctx, Map runs per record with the backend already pointed at that | ||
| 12 | // record's key. | ||
| 13 | type RichMapFunc[IN, OUT any] interface { | ||
| 14 | Open(ctx context.Context) error | ||
| 15 | Map(ctx context.Context, in IN) (OUT, error) | ||
| 16 | Close(ctx context.Context) error | ||
| 17 | } | ||
| 18 | |||
| 19 | type StatefulMapOperator[IN, OUT any] struct { | ||
| 20 | userFunc RichMapFunc[IN, OUT] | ||
| 21 | backend core.StateBackend | ||
| 22 | } | ||
| 23 | |||
| 24 | func NewStatefulMapOperator[IN, OUT any](fn RichMapFunc[IN, OUT]) *StatefulMapOperator[IN, OUT] { | ||
| 25 | return &StatefulMapOperator[IN, OUT]{userFunc: fn} | ||
| 26 | } | ||
| 27 | |||
| 28 | func (s *StatefulMapOperator[IN, OUT]) Open(ctx context.Context) error { | ||
| 29 | // Get backend state from context via using context KV | ||
| 30 | be, ok := core.ExtractStateBackend(ctx) | ||
| 31 | if !ok || be == nil { | ||
| 32 | return fmt.Errorf("stateful map: no state backend in context") | ||
| 33 | } | ||
| 34 | |||
| 35 | s.backend = be | ||
| 36 | // The user func opens its ValueState off the same ctx. | ||
| 37 | return s.userFunc.Open(ctx) | ||
| 38 | } | ||
| 39 | |||
| 40 | func (s *StatefulMapOperator[IN, OUT]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error { | ||
| 41 | if record.Type != core.RecordData { | ||
| 42 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 43 | Type: record.Type, | ||
| 44 | Timestamp: record.Timestamp, | ||
| 45 | Key: record.Key, | ||
| 46 | BarrierID: record.BarrierID, | ||
| 47 | }) | ||
| 48 | } | ||
| 49 | |||
| 50 | // Without a key every record would silently share the "" state slot. | ||
| 51 | if record.Key == "" { | ||
| 52 | return fmt.Errorf("stateful map: record %v has no key, keyBy it first", record.Value) | ||
| 53 | } | ||
| 54 | |||
| 55 | if err := s.backend.SetCurrentKey(record.Key); err != nil { | ||
| 56 | return fmt.Errorf("set current key %q: %w", record.Key, err) | ||
| 57 | } | ||
| 58 | |||
| 59 | // run the user transform | ||
| 60 | res, err := s.userFunc.Map(ctx, record.Value) | ||
| 61 | if err != nil { | ||
| 62 | return fmt.Errorf("cannot process value %v: %w", record.Value, err) | ||
| 63 | } | ||
| 64 | |||
| 65 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 66 | Type: core.RecordData, | ||
| 67 | Timestamp: record.Timestamp, | ||
| 68 | Key: record.Key, | ||
| 69 | Value: res, | ||
| 70 | }) | ||
| 71 | } | ||
| 72 | |||
| 73 | func (s *StatefulMapOperator[IN, OUT]) Close(ctx context.Context) error { | ||
| 74 | return s.userFunc.Close(ctx) | ||
| 75 | } | ||
diff --git a/core/operator/watermark.go b/core/operator/watermark.go new file mode 100644 index 0000000..3d591e5 --- /dev/null +++ b/core/operator/watermark.go | |||
| @@ -0,0 +1,61 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | |||
| 6 | "flux/core" | ||
| 7 | ) | ||
| 8 | |||
| 9 | // TimestampAssigner is responsible for event time defined by user | ||
| 10 | type TimestampAssigner[IN any] func(in IN) int64 | ||
| 11 | |||
| 12 | type WaterGeneratorOperator[IN any] struct { | ||
| 13 | assigner TimestampAssigner[IN] | ||
| 14 | maxOutOfOrder int64 | ||
| 15 | currentMaxTimestamp int64 | ||
| 16 | lastEmittedWatermark int64 | ||
| 17 | } | ||
| 18 | |||
| 19 | func NewWatermarkGeneratorOperator[IN any](maxOutOfOrder int64, assigner TimestampAssigner[IN]) *WaterGeneratorOperator[IN] { | ||
| 20 | return &WaterGeneratorOperator[IN]{ | ||
| 21 | maxOutOfOrder: maxOutOfOrder, | ||
| 22 | assigner: assigner, | ||
| 23 | lastEmittedWatermark: -1, | ||
| 24 | } | ||
| 25 | } | ||
| 26 | |||
| 27 | func (w *WaterGeneratorOperator[IN]) Open(ctx context.Context) error { | ||
| 28 | return nil | ||
| 29 | } | ||
| 30 | |||
| 31 | func (w *WaterGeneratorOperator[IN]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[IN]) error { | ||
| 32 | if record.Type == core.RecordData { | ||
| 33 | if err := out.Emit(ctx, record); err != nil { | ||
| 34 | return err | ||
| 35 | } | ||
| 36 | // query the time | ||
| 37 | eventTime := w.assigner(record.Value) | ||
| 38 | record.Timestamp = eventTime | ||
| 39 | |||
| 40 | // update the max timestamp | ||
| 41 | if w.currentMaxTimestamp < eventTime { | ||
| 42 | w.currentMaxTimestamp = eventTime | ||
| 43 | } | ||
| 44 | |||
| 45 | // calc the watermark | ||
| 46 | |||
| 47 | watermark := w.currentMaxTimestamp - w.maxOutOfOrder | ||
| 48 | |||
| 49 | // check if current watermark later than previous watermark | ||
| 50 | if watermark > w.lastEmittedWatermark { | ||
| 51 | w.lastEmittedWatermark = watermark | ||
| 52 | return out.Emit(ctx, core.NewWatermarkRecord[IN](watermark)) | ||
| 53 | } | ||
| 54 | return nil | ||
| 55 | } | ||
| 56 | return out.Emit(ctx, record) | ||
| 57 | } | ||
| 58 | |||
| 59 | func (w *WaterGeneratorOperator[IN]) Close(ctx context.Context) error { | ||
| 60 | return nil | ||
| 61 | } | ||
diff --git a/core/operator/window.go b/core/operator/window.go new file mode 100644 index 0000000..610f6e3 --- /dev/null +++ b/core/operator/window.go | |||
| @@ -0,0 +1,110 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "fmt" | ||
| 6 | |||
| 7 | "flux/core" | ||
| 8 | ) | ||
| 9 | |||
| 10 | // WindowFunction is for user interface implements the behaviour | ||
| 11 | type WindowFunction[IN, OUT any] interface { | ||
| 12 | Apply(ctx context.Context, key string, window core.Window, in []IN, out Emitter[OUT]) error | ||
| 13 | } | ||
| 14 | |||
| 15 | type WindowOperator[IN, OUT any] struct { | ||
| 16 | assigner core.WindowAssigner[IN] | ||
| 17 | userfunc WindowFunction[IN, OUT] | ||
| 18 | backend core.StateBackend | ||
| 19 | timeSerivce core.InternalTimerService | ||
| 20 | |||
| 21 | windowState core.ValueState[[]IN] | ||
| 22 | } | ||
| 23 | |||
| 24 | func NewWindowOperator[IN, OUT any](assigner core.WindowAssigner[IN], userfunc WindowFunction[IN, OUT]) *WindowOperator[IN, OUT] { | ||
| 25 | return &WindowOperator[IN, OUT]{ | ||
| 26 | assigner: assigner, | ||
| 27 | userfunc: userfunc, | ||
| 28 | timeSerivce: core.NewTimerService(), | ||
| 29 | } | ||
| 30 | } | ||
| 31 | |||
| 32 | func (o *WindowOperator[IN, OUT]) Open(ctx context.Context) error { | ||
| 33 | be, ok := core.ExtractStateBackend(ctx) | ||
| 34 | if !ok { | ||
| 35 | return fmt.Errorf("cannot extract state backend, got: %v", be) | ||
| 36 | } | ||
| 37 | o.backend = be | ||
| 38 | windowState, err := be.GetValueState(ctx, "window-elements") | ||
| 39 | if err != nil { | ||
| 40 | return fmt.Errorf("cannot get value state window-elements error: %w", err) | ||
| 41 | } | ||
| 42 | |||
| 43 | o.windowState = core.CastValueType[[]IN](windowState) | ||
| 44 | |||
| 45 | return nil | ||
| 46 | } | ||
| 47 | |||
| 48 | func (o *WindowOperator[IN, OUT]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error { | ||
| 49 | switch record.Type { | ||
| 50 | case core.RecordData: | ||
| 51 | // current windows state will hold all record in window | ||
| 52 | // assign and get list window | ||
| 53 | windows := o.assigner.AssignWindows(record.Value, record.Timestamp) | ||
| 54 | for _, w := range windows { | ||
| 55 | o.backend.SetCurrentNamespace(w) | ||
| 56 | o.backend.SetCurrentKey(record.Key) | ||
| 57 | o.timeSerivce.Register(w.MaxTimestamp(), record.Key, w) | ||
| 58 | |||
| 59 | currentWindowState, err := o.windowState.Value(ctx) | ||
| 60 | if err != nil { | ||
| 61 | return err | ||
| 62 | } | ||
| 63 | |||
| 64 | currentWindowState = append(currentWindowState, record.Value) | ||
| 65 | |||
| 66 | o.windowState.Update(ctx, currentWindowState) | ||
| 67 | } | ||
| 68 | |||
| 69 | return nil | ||
| 70 | case core.RecordWatermark: | ||
| 71 | deadlines := o.timeSerivce.AdvanceWatermark(record.Timestamp) | ||
| 72 | |||
| 73 | for _, timer := range deadlines { | ||
| 74 | o.backend.SetCurrentKey(timer.Key) | ||
| 75 | o.backend.SetCurrentNamespace(timer.Namespace) | ||
| 76 | |||
| 77 | currentWindowState, err := o.windowState.Value(ctx) | ||
| 78 | if err != nil { | ||
| 79 | return err | ||
| 80 | } | ||
| 81 | |||
| 82 | // NOTE: this is workaround create inline window, refactor later | ||
| 83 | |||
| 84 | if err := o.userfunc.Apply(ctx, timer.Key, timer.Namespace.(core.Window), currentWindowState, out); err != nil { | ||
| 85 | return err | ||
| 86 | } | ||
| 87 | |||
| 88 | if err := o.windowState.Clear(ctx); err != nil { | ||
| 89 | return err | ||
| 90 | } | ||
| 91 | |||
| 92 | } | ||
| 93 | |||
| 94 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 95 | Timestamp: record.Timestamp, | ||
| 96 | Type: core.RecordWatermark, | ||
| 97 | }) | ||
| 98 | default: | ||
| 99 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 100 | BarrierID: record.BarrierID, | ||
| 101 | Timestamp: record.Timestamp, | ||
| 102 | Key: record.Key, | ||
| 103 | Type: record.Type, | ||
| 104 | }) | ||
| 105 | } | ||
| 106 | } | ||
| 107 | |||
| 108 | func (o *WindowOperator[IN, OUT]) Close(ctx context.Context) error { | ||
| 109 | return nil | ||
| 110 | } | ||
diff --git a/core/record.go b/core/record.go new file mode 100644 index 0000000..e18e58d --- /dev/null +++ b/core/record.go | |||
| @@ -0,0 +1,40 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | type RecordType byte | ||
| 4 | |||
| 5 | const ( | ||
| 6 | RecordData RecordType = iota | ||
| 7 | RecordWatermark | ||
| 8 | RecordBarrier | ||
| 9 | ) | ||
| 10 | |||
| 11 | type StreamRecord[T any] struct { | ||
| 12 | Value T | ||
| 13 | Key string // for keyBy stateful processing | ||
| 14 | Timestamp int64 // epoch ms time | ||
| 15 | BarrierID uint64 // ID checkpoint barrier | ||
| 16 | Type RecordType | ||
| 17 | } | ||
| 18 | |||
| 19 | func NewDataRecord[T any](key string, val T, timestamp int64) StreamRecord[T] { | ||
| 20 | return StreamRecord[T]{ | ||
| 21 | Value: val, | ||
| 22 | Key: key, | ||
| 23 | Timestamp: timestamp, | ||
| 24 | Type: RecordData, | ||
| 25 | } | ||
| 26 | } | ||
| 27 | |||
| 28 | func NewWatermarkRecord[T any](timestamp int64) StreamRecord[T] { | ||
| 29 | return StreamRecord[T]{ | ||
| 30 | Type: RecordWatermark, | ||
| 31 | Timestamp: timestamp, | ||
| 32 | } | ||
| 33 | } | ||
| 34 | |||
| 35 | func NewBarrierRecord[T any](barrierID uint64) StreamRecord[T] { | ||
| 36 | return StreamRecord[T]{ | ||
| 37 | Type: RecordBarrier, | ||
| 38 | BarrierID: barrierID, | ||
| 39 | } | ||
| 40 | } | ||
diff --git a/core/state.go b/core/state.go new file mode 100644 index 0000000..d6d7847 --- /dev/null +++ b/core/state.go | |||
| @@ -0,0 +1,10 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | import "context" | ||
| 4 | |||
| 5 | // ValueState is for client side using to update state | ||
| 6 | type ValueState[T any] interface { | ||
| 7 | Value(ctx context.Context) (T, error) | ||
| 8 | Update(ctx context.Context, newVal T) (T, error) | ||
| 9 | Clear(ctx context.Context) error | ||
| 10 | } | ||
diff --git a/core/state_backend.go b/core/state_backend.go new file mode 100644 index 0000000..9f9f397 --- /dev/null +++ b/core/state_backend.go | |||
| @@ -0,0 +1,62 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | ) | ||
| 6 | |||
| 7 | type StateBackend interface { | ||
| 8 | // For Framework === | ||
| 9 | // SetCurrentKey will perform in Memory | ||
| 10 | SetCurrentKey(key string) error | ||
| 11 | // GetCurrentKey will perform in Memory | ||
| 12 | GetCurrentKey() string | ||
| 13 | SetCurrentNamespace(namespace any) error | ||
| 14 | GetCurrentNamespace() any | ||
| 15 | |||
| 16 | // For User === | ||
| 17 | // GetValueState will perform in disk | ||
| 18 | GetValueState(ctx context.Context, key string) (ValueState[any], error) | ||
| 19 | } | ||
| 20 | |||
| 21 | type stateBackendKeyType struct{} | ||
| 22 | |||
| 23 | var stateBackendKey stateBackendKeyType | ||
| 24 | |||
| 25 | type typedValueState[T any] struct { | ||
| 26 | inner ValueState[any] | ||
| 27 | } | ||
| 28 | |||
| 29 | func CastValueType[T any](inner ValueState[any]) ValueState[T] { | ||
| 30 | return &typedValueState[T]{inner: inner} | ||
| 31 | } | ||
| 32 | |||
| 33 | func (s *typedValueState[T]) Value(ctx context.Context) (T, error) { | ||
| 34 | val, err := s.inner.Value(ctx) | ||
| 35 | if err != nil { | ||
| 36 | var zero T | ||
| 37 | return zero, err | ||
| 38 | } | ||
| 39 | if val == nil { | ||
| 40 | var zero T | ||
| 41 | return zero, nil | ||
| 42 | } | ||
| 43 | return val.(T), nil | ||
| 44 | } | ||
| 45 | |||
| 46 | func (s *typedValueState[T]) Update(ctx context.Context, newVal T) (T, error) { | ||
| 47 | _, err := s.inner.Update(ctx, newVal) | ||
| 48 | return newVal, err | ||
| 49 | } | ||
| 50 | |||
| 51 | func (s *typedValueState[T]) Clear(ctx context.Context) error { | ||
| 52 | return s.inner.Clear(ctx) | ||
| 53 | } | ||
| 54 | |||
| 55 | func InjectStateBackend(ctx context.Context, state StateBackend) context.Context { | ||
| 56 | return context.WithValue(ctx, stateBackendKey, state) | ||
| 57 | } | ||
| 58 | |||
| 59 | func ExtractStateBackend(ctx context.Context) (StateBackend, bool) { | ||
| 60 | backend, ok := ctx.Value(stateBackendKey).(StateBackend) | ||
| 61 | return backend, ok | ||
| 62 | } | ||
diff --git a/core/timer.go b/core/timer.go new file mode 100644 index 0000000..3d17299 --- /dev/null +++ b/core/timer.go | |||
| @@ -0,0 +1,113 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "container/heap" | ||
| 5 | "context" | ||
| 6 | ) | ||
| 7 | |||
| 8 | // If using rockDB no need priority queue anymore | ||
| 9 | // implement priority queue from golang | ||
| 10 | // ref: https://pkg.go.dev/container/heap#example-package-PriorityQueue | ||
| 11 | |||
| 12 | type timerServiceKey struct{} | ||
| 13 | |||
| 14 | // InjectTimerService inject the timer into Context | ||
| 15 | func InjectTimerService(ctx context.Context, ts TimerService) context.Context { | ||
| 16 | return context.WithValue(ctx, timerServiceKey{}, ts) | ||
| 17 | } | ||
| 18 | |||
| 19 | // ExtractTimerService get the timer for user from context | ||
| 20 | func ExtractTimerService(ctx context.Context) (TimerService, bool) { | ||
| 21 | ts, ok := ctx.Value(timerServiceKey{}).(TimerService) | ||
| 22 | return ts, ok | ||
| 23 | } | ||
| 24 | |||
| 25 | type Timer struct { | ||
| 26 | // Timestamp is the time when Timer is triggered | ||
| 27 | Timestamp int64 | ||
| 28 | // Key is for who User involve | ||
| 29 | Key string | ||
| 30 | // Window is for time range start-end of | ||
| 31 | // String actually cost more bytes than Window(16bytes) | ||
| 32 | // and we dont want to decode from namespace to Window | ||
| 33 | Namespace any | ||
| 34 | } | ||
| 35 | |||
| 36 | // Priority Queue (Min Heap). Smallest timestamp will be on top | ||
| 37 | |||
| 38 | type timerHeap []Timer | ||
| 39 | |||
| 40 | func (h *timerHeap) Len() int { return len(*h) } | ||
| 41 | func (h *timerHeap) Less(i, j int) bool { return (*h)[i].Timestamp < (*h)[j].Timestamp } | ||
| 42 | func (h *timerHeap) Swap(i, j int) { | ||
| 43 | (*h)[i], (*h)[j] = (*h)[j], (*h)[i] | ||
| 44 | } | ||
| 45 | |||
| 46 | func (h *timerHeap) Push(x any) { | ||
| 47 | *h = append(*h, x.(Timer)) | ||
| 48 | } | ||
| 49 | |||
| 50 | func (h *timerHeap) Pop() any { | ||
| 51 | old := *h | ||
| 52 | n := len(old) | ||
| 53 | last := old[n-1] | ||
| 54 | *h = old[:n-1] | ||
| 55 | return last | ||
| 56 | } | ||
| 57 | |||
| 58 | type TimerService interface { | ||
| 59 | Register(time int64, key string, keyspace any) | ||
| 60 | CurrentWatermark() int64 | ||
| 61 | } | ||
| 62 | |||
| 63 | type InternalTimerService interface { | ||
| 64 | TimerService | ||
| 65 | AdvanceWatermark(watermark int64) []Timer | ||
| 66 | } | ||
| 67 | |||
| 68 | type timerService struct { | ||
| 69 | queue timerHeap | ||
| 70 | currentWatermark int64 | ||
| 71 | } | ||
| 72 | |||
| 73 | func NewTimerService() InternalTimerService { | ||
| 74 | return &timerService{ | ||
| 75 | queue: make(timerHeap, 0), | ||
| 76 | currentWatermark: -1, | ||
| 77 | } | ||
| 78 | } | ||
| 79 | |||
| 80 | func (t *timerService) Register(time int64, key string, namespace any) { | ||
| 81 | heap.Push(&t.queue, Timer{ | ||
| 82 | Timestamp: time, | ||
| 83 | Key: key, | ||
| 84 | Namespace: namespace, | ||
| 85 | }) | ||
| 86 | } | ||
| 87 | |||
| 88 | func (t *timerService) CurrentWatermark() int64 { | ||
| 89 | return t.currentWatermark | ||
| 90 | } | ||
| 91 | |||
| 92 | func (t *timerService) AdvanceWatermark(watermark int64) []Timer { | ||
| 93 | if watermark < t.currentWatermark { | ||
| 94 | // no need to advance when get smaller | ||
| 95 | return nil | ||
| 96 | } | ||
| 97 | |||
| 98 | t.currentWatermark = watermark | ||
| 99 | |||
| 100 | var timers []Timer | ||
| 101 | for t.queue.Len() > 0 { | ||
| 102 | earliestTimer := t.queue[0] | ||
| 103 | |||
| 104 | if earliestTimer.Timestamp > watermark { | ||
| 105 | break | ||
| 106 | } | ||
| 107 | |||
| 108 | pop := heap.Pop(&t.queue).(Timer) | ||
| 109 | timers = append(timers, pop) | ||
| 110 | } | ||
| 111 | |||
| 112 | return timers | ||
| 113 | } | ||
diff --git a/core/window.go b/core/window.go new file mode 100644 index 0000000..6c4b98f --- /dev/null +++ b/core/window.go | |||
| @@ -0,0 +1,54 @@ | |||
| 1 | package core | ||
| 2 | |||
| 3 | import "fmt" | ||
| 4 | |||
| 5 | type Window interface { | ||
| 6 | // MaxTimestamp is the maximum time the Window is Open | ||
| 7 | MaxTimestamp() int64 | ||
| 8 | // ToString convert Window to string format | ||
| 9 | ToString() string | ||
| 10 | } | ||
| 11 | |||
| 12 | type TimeWindow struct { | ||
| 13 | Start int64 | ||
| 14 | End int64 | ||
| 15 | } | ||
| 16 | |||
| 17 | func (t TimeWindow) MaxTimestamp() int64 { | ||
| 18 | // MaxTimestamp is 1ms before the End | ||
| 19 | return t.End - 1 | ||
| 20 | } | ||
| 21 | |||
| 22 | func (t TimeWindow) ToString() string { | ||
| 23 | return fmt.Sprintf("timewindow(%d,%d)", t.Start, t.End) | ||
| 24 | } | ||
| 25 | |||
| 26 | type WindowAssigner[IN any] interface { | ||
| 27 | AssignWindows(element IN, timestamp int64) []Window | ||
| 28 | } | ||
| 29 | |||
| 30 | type TumblingWindows[IN any] struct { | ||
| 31 | Size int64 | ||
| 32 | } | ||
| 33 | |||
| 34 | func NewTumblingWindows[IN any](size int64) *TumblingWindows[IN] { | ||
| 35 | return &TumblingWindows[IN]{ | ||
| 36 | Size: size, | ||
| 37 | } | ||
| 38 | } | ||
| 39 | |||
| 40 | func (w *TumblingWindows[IN]) AssignWindows(element IN, timestamp int64) []Window { | ||
| 41 | // module to get start end | ||
| 42 | // if timestamp = 1200, size 1000 | ||
| 43 | // so start will 1200 - 200 = 1000, end will 1000 + 1000 = 2000 | ||
| 44 | start := timestamp - (timestamp % w.Size) | ||
| 45 | |||
| 46 | end := start + w.Size | ||
| 47 | |||
| 48 | return []Window{ | ||
| 49 | TimeWindow{ | ||
| 50 | Start: start, | ||
| 51 | End: end, | ||
| 52 | }, | ||
| 53 | } | ||
| 54 | } | ||
