diff options
Diffstat (limited to 'core/operator/stateful_map.go')
| -rw-r--r-- | core/operator/stateful_map.go | 75 |
1 files changed, 75 insertions, 0 deletions
diff --git a/core/operator/stateful_map.go b/core/operator/stateful_map.go new file mode 100644 index 0000000..295c656 --- /dev/null +++ b/core/operator/stateful_map.go | |||
| @@ -0,0 +1,75 @@ | |||
| 1 | package operator | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "fmt" | ||
| 6 | |||
| 7 | "flux/core" | ||
| 8 | ) | ||
| 9 | |||
| 10 | // RichMapFunc is a stateful map on the user side: Open grabs the state handles | ||
| 11 | // off ctx, Map runs per record with the backend already pointed at that | ||
| 12 | // record's key. | ||
| 13 | type RichMapFunc[IN, OUT any] interface { | ||
| 14 | Open(ctx context.Context) error | ||
| 15 | Map(ctx context.Context, in IN) (OUT, error) | ||
| 16 | Close(ctx context.Context) error | ||
| 17 | } | ||
| 18 | |||
| 19 | type StatefulMapOperator[IN, OUT any] struct { | ||
| 20 | userFunc RichMapFunc[IN, OUT] | ||
| 21 | backend core.StateBackend | ||
| 22 | } | ||
| 23 | |||
| 24 | func NewStatefulMapOperator[IN, OUT any](fn RichMapFunc[IN, OUT]) *StatefulMapOperator[IN, OUT] { | ||
| 25 | return &StatefulMapOperator[IN, OUT]{userFunc: fn} | ||
| 26 | } | ||
| 27 | |||
| 28 | func (s *StatefulMapOperator[IN, OUT]) Open(ctx context.Context) error { | ||
| 29 | // Get backend state from context via using context KV | ||
| 30 | be, ok := core.ExtractStateBackend(ctx) | ||
| 31 | if !ok || be == nil { | ||
| 32 | return fmt.Errorf("stateful map: no state backend in context") | ||
| 33 | } | ||
| 34 | |||
| 35 | s.backend = be | ||
| 36 | // The user func opens its ValueState off the same ctx. | ||
| 37 | return s.userFunc.Open(ctx) | ||
| 38 | } | ||
| 39 | |||
| 40 | func (s *StatefulMapOperator[IN, OUT]) Process(ctx context.Context, record core.StreamRecord[IN], out Emitter[OUT]) error { | ||
| 41 | if record.Type != core.RecordData { | ||
| 42 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 43 | Type: record.Type, | ||
| 44 | Timestamp: record.Timestamp, | ||
| 45 | Key: record.Key, | ||
| 46 | BarrierID: record.BarrierID, | ||
| 47 | }) | ||
| 48 | } | ||
| 49 | |||
| 50 | // Without a key every record would silently share the "" state slot. | ||
| 51 | if record.Key == "" { | ||
| 52 | return fmt.Errorf("stateful map: record %v has no key, keyBy it first", record.Value) | ||
| 53 | } | ||
| 54 | |||
| 55 | if err := s.backend.SetCurrentKey(record.Key); err != nil { | ||
| 56 | return fmt.Errorf("set current key %q: %w", record.Key, err) | ||
| 57 | } | ||
| 58 | |||
| 59 | // run the user transform | ||
| 60 | res, err := s.userFunc.Map(ctx, record.Value) | ||
| 61 | if err != nil { | ||
| 62 | return fmt.Errorf("cannot process value %v: %w", record.Value, err) | ||
| 63 | } | ||
| 64 | |||
| 65 | return out.Emit(ctx, core.StreamRecord[OUT]{ | ||
| 66 | Type: core.RecordData, | ||
| 67 | Timestamp: record.Timestamp, | ||
| 68 | Key: record.Key, | ||
| 69 | Value: res, | ||
| 70 | }) | ||
| 71 | } | ||
| 72 | |||
| 73 | func (s *StatefulMapOperator[IN, OUT]) Close(ctx context.Context) error { | ||
| 74 | return s.userFunc.Close(ctx) | ||
| 75 | } | ||
