blob: 74b08f07072c1528765b3d7aee6079603e63bc5e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
package pipeline
import (
"context"
"time"
)
// Batch collects items into slices of up to size, flushing when full or when timeout elapses.
func Batch[T any](ctx context.Context, in <-chan T, size int, timeout time.Duration) <-chan []T {
if in == nil {
panic("pipeline: Batch input channel must not be nil")
}
if size < 1 {
panic("pipeline: Batch size must be at least 1")
}
out := make(chan []T)
go func() {
defer close(out)
buf := make([]T, 0, size)
ticker := time.NewTicker(timeout)
defer ticker.Stop()
flush := func() bool {
if len(buf) == 0 {
return true
}
batch := buf
buf = make([]T, 0, size)
select {
case out <- batch:
return true
case <-ctx.Done():
return false
}
}
for {
select {
case v, ok := <-in:
if !ok {
flush()
return
}
buf = append(buf, v)
if len(buf) >= size {
if !flush() {
return
}
}
case <-ticker.C:
if !flush() {
return
}
case <-ctx.Done():
return
}
}
}()
return out
}
|
