Skip to content

StreamBatchSubscriber

faststream.redis.subscriber.usecases.stream_subscriber.StreamBatchSubscriber #

StreamBatchSubscriber(
    config: RedisSubscriberConfig,
    specification: SubscriberSpecification[Any, Any],
    calls: CallsCollection[Any],
)

Bases: _StreamHandlerMixin

Source code in faststream/redis/subscriber/usecases/stream_subscriber.py
476
477
478
479
480
481
482
483
484
485
def __init__(
    self,
    config: "RedisSubscriberConfig",
    specification: "SubscriberSpecification[Any, Any]",
    calls: "CallsCollection[Any]",
) -> None:
    parser = RedisBatchStreamParser(config)
    config.decoder = parser.decode_message
    config.parser = parser.parse_message
    super().__init__(config, specification, calls)

lock instance-attribute #

lock: AbstractContextManager[Any] = FakeContext()

extra_watcher_options instance-attribute #

extra_watcher_options: dict[str, Any] = {}

graceful_timeout instance-attribute #

graceful_timeout: float | None

calls instance-attribute #

calls = calls

specification instance-attribute #

specification = specification

ack_policy instance-attribute #

ack_policy = config.ack_policy

running instance-attribute #

running = False

tasks instance-attribute #

tasks: list[Task[Any]] = []

config instance-attribute #

config = config

last_id instance-attribute #

last_id = config.stream_sub.last_id

read_id instance-attribute #

read_id = self.last_id

min_idle_time instance-attribute #

min_idle_time = config.stream_sub.min_idle_time

claim_min_idle_time instance-attribute #

claim_min_idle_time = config.stream_sub.claim_min_idle_time

autoclaim_start_id instance-attribute #

autoclaim_start_id = b'0-0'

stream_sub property #

stream_sub: StreamSub

start async #

start() -> None
Source code in faststream/redis/subscriber/usecases/stream_subscriber.py
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
@override
async def start(self) -> None:
    client = self._client

    self.extra_watcher_options.update(
        redis=client,
        group=self.stream_sub.group,
    )

    stream = self.stream_sub

    read: ReadCallable

    if stream.group and stream.consumer:
        group_create_id = "$" if self.last_id == ">" else self.last_id
        try:
            await client.xgroup_create(
                name=stream.name,
                id=group_create_id,
                groupname=stream.group,
                mkstream=stream.declare,
            )
        except ResponseError as e:
            if "already exists" not in str(e):
                raise
        else:
            self.read_id = ">"

        self.last_id = self.read_id

        if stream.min_idle_time is None:

            def read(_: str) -> Awaitable[ReadResponse]:
                return self._xreadgroup(
                    count=stream.max_records,
                    block=stream.polling_interval,
                    noack=stream.no_ack,
                )

        else:

            async def read(_: str) -> ReadResponse:
                stream_message = await client.xautoclaim(
                    name=self.stream_sub.name,
                    groupname=self.stream_sub.group,
                    consumername=self.stream_sub.consumer,
                    min_idle_time=self.min_idle_time,
                    start_id=self.autoclaim_start_id,
                    count=1,
                )
                stream_name = self.stream_sub.name.encode()
                (next_id, messages, *_) = stream_message

                # Update start_id for next call
                self.autoclaim_start_id = next_id

                if next_id == b"0-0" and not messages:
                    await asyncio.sleep(stream.polling_interval / 1000)  # ms to s
                    return ()

                return ((stream_name, messages),)

    else:

        def read(
            last_id: str,
        ) -> Awaitable[ReadResponse]:
            return client.xread(
                {stream.name: last_id},
                block=stream.polling_interval,
                count=stream.max_records,
            )

    await super().start(read)

stop async #

stop() -> None

Clean up handler subscription, cancel consume task in graceful mode.

Source code in faststream/_internal/endpoint/subscriber/mixins.py
43
44
45
46
47
48
49
50
51
async def stop(self) -> None:
    """Clean up handler subscription, cancel consume task in graceful mode."""
    await super().stop()

    for task in self.tasks:
        if not task.done():
            task.cancel()

    self.tasks.clear()

add_call #

add_call(
    *,
    parser_: Optional[CustomCallable],
    decoder_: Optional[CustomCallable],
    dependencies_: Sequence[Dependant],
    codec_: Optional[CodecProto] = None,
) -> Self
Source code in faststream/_internal/endpoint/subscriber/usecase.py
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
def add_call(
    self,
    *,
    parser_: Optional["CustomCallable"],
    decoder_: Optional["CustomCallable"],
    dependencies_: Sequence["Dependant"],
    codec_: Optional["CodecProto"] = None,
) -> Self:
    self._call_options = _CallOptions(
        parser=parser_,
        decoder=decoder_,
        dependencies=dependencies_,
        codec=codec_,
    )
    return self

consume async #

consume(msg: MsgType) -> Any

Consume a message asynchronously.

Source code in faststream/_internal/endpoint/subscriber/usecase.py
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
async def consume(self, msg: MsgType) -> Any:
    """Consume a message asynchronously."""
    if not self.running:
        return None

    try:
        return await self.process_message(msg)

    except StopConsume:
        # Stop handler at StopConsume exception
        await self.stop()

    except SystemExit:
        # Stop handler at `exit()` call
        await self.stop()

        if app := self._outer_config.context.get("app"):
            app.exit()

    except Exception:  # nosec B110  # noqa: S110
        # All other exceptions were logged by CriticalLogMiddleware
        pass

process_message async #

process_message(msg: MsgType) -> Response

Execute all message processing stages.

Source code in faststream/_internal/endpoint/subscriber/usecase.py
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
async def process_message(self, msg: MsgType) -> "Response":
    """Execute all message processing stages."""
    context = self._outer_config.context
    logger_state = self._outer_config.logger

    async with AsyncExitStack() as stack:
        stack.enter_context(self.lock)

        # Enter context before middlewares
        stack.enter_context(
            context.scopes(
                (
                    ("handler_", self),
                    ("logger", logger_state.logger.logger),
                    *self._outer_config.extra_context.items(),
                ),
            ),
        )

        # enter all middlewares
        middlewares: list[BaseMiddleware] = []
        for base_m in self.__build__middlewares_stack():
            middleware = base_m(msg, context=context)
            middlewares.append(middleware)
            await middleware.__aenter__()

        cache: dict[Any, Any] = {}
        parsing_error: Exception | None = None
        for h in self.calls:
            try:
                message = await h.is_suitable(msg, cache)
            except Exception as e:
                parsing_error = e
                break

            if message is not None:
                stack.enter_context(
                    context.scopes(
                        (
                            ("log_context", self.get_log_context(message)),
                            ("message", message),
                        ),
                    ),
                )

                # Middlewares should be exited before scope release
                for m in middlewares:
                    stack.push_async_exit(m.__aexit__)

                result_msg = ensure_response(
                    await h.call(
                        message=message,
                        # consumer middlewares
                        _extra_middlewares=(
                            m.consume_scope for m in middlewares[::-1]
                        ),
                    ),
                )

                if not result_msg.correlation_id:
                    result_msg.correlation_id = message.correlation_id

                for p in chain(
                    self.__get_response_publisher(message),
                    h.handler._publishers,
                ):
                    await p._publish(
                        result_msg.as_publish_command(),
                        _extra_middlewares=(
                            m.publish_scope for m in middlewares[::-1]
                        ),
                    )

                # Return data for tests
                return result_msg

        # Suitable handler was not found or
        # parsing/decoding exception occurred
        for m in middlewares:
            stack.push_async_exit(m.__aexit__)

        # Reraise it to catch in tests
        if parsing_error:
            raise parsing_error

        error_msg = f"There is no suitable handler for {msg=}"
        raise SubscriberNotFound(error_msg)

    # An error was raised and processed by some middleware
    return ensure_response(None)

get_one async #

get_one(
    *, timeout: float = 5.0
) -> RedisStreamMessage | None
Source code in faststream/redis/subscriber/usecases/stream_subscriber.py
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
@override
async def get_one(
    self,
    *,
    timeout: float = 5.0,
) -> "RedisStreamMessage | None":
    assert not self.calls, (
        "You can't use `get_one` method if subscriber has registered handlers."
    )
    claim_meta: ClaimMeta | None = None

    if self.stream_sub.group and self.stream_sub.consumer:
        if self.min_idle_time is None:
            stream_message = await self._xreadgroup(
                count=1,
                block=math.ceil(timeout * 1000),
            )
            if not stream_message:
                return None

            ((stream_name, (entry,)),) = stream_message
            message_id, raw_message, claim_meta = self._parse_stream_entry(entry)
        else:
            stream_message = await self._client.xautoclaim(
                name=self.stream_sub.name,
                groupname=self.stream_sub.group,
                consumername=self.stream_sub.consumer,
                min_idle_time=self.min_idle_time,
                start_id=self.autoclaim_start_id,
                count=1,
            )
            (next_id, messages, *_) = stream_message
            # Update start_id for next call
            self.autoclaim_start_id = next_id
            if not messages:
                return None
            stream_name = self.stream_sub.name.encode()
            ((message_id, raw_message),) = messages
    else:
        stream_message = await self._client.xread(
            {self.stream_sub.name: self.last_id},
            block=math.ceil(timeout * 1000),
            count=1,
        )
        if not stream_message:
            return None

        ((stream_name, ((message_id, raw_message),)),) = stream_message

    self.last_id = message_id.decode()

    redis_incoming_msg = DefaultStreamMessage(
        type="stream",
        channel=stream_name.decode(),
        message_ids=[message_id],
        data=raw_message,
    )
    _attach_claim_metadata(redis_incoming_msg, [claim_meta])

    context = self._outer_config.context
    async_parser, async_decoder = self._get_parser_and_decoder()

    msg: RedisStreamMessage = await process_msg(  # type: ignore[assignment]
        msg=redis_incoming_msg,
        middlewares=(
            m(redis_incoming_msg, context=context) for m in self._broker_middlewares
        ),
        parser=async_parser,
        decoder=async_decoder,
    )
    return msg

get_log_context #

get_log_context(
    message: Optional[StreamMessage[Any]],
) -> dict[str, str]
Source code in faststream/redis/subscriber/usecases/stream_subscriber.py
92
93
94
95
96
97
98
99
def get_log_context(
    self,
    message: Optional["BrokerStreamMessage[Any]"],
) -> dict[str, str]:
    return self.build_log_context(
        message=message,
        channel=self.stream_sub.name,
    )

schema #

schema() -> dict[str, SubscriberSpec]
Source code in faststream/_internal/endpoint/subscriber/usecase.py
503
504
505
def schema(self) -> dict[str, "SubscriberSpec"]:
    self._build_fastdepends_model()
    return self.specification.get_schema()

add_task #

add_task(
    func: Callable[..., Coroutine[Any, Any, Any]],
    func_args: tuple[Any, ...] | None = None,
    func_kwargs: dict[str, Any] | None = None,
    *,
    restart_on_failure: bool = True,
) -> None
Source code in faststream/_internal/endpoint/subscriber/mixins.py
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def add_task(
    self,
    func: Callable[..., Coroutine[Any, Any, Any]],
    func_args: tuple[Any, ...] | None = None,
    func_kwargs: dict[str, Any] | None = None,
    *,
    restart_on_failure: bool = True,
) -> None:
    args = func_args or ()
    kwargs = func_kwargs or {}
    task = asyncio.create_task(func(*args, **kwargs))
    callback = TaskCallbackSupervisor(
        func,
        func_args,
        func_kwargs,
        self,
        restart_on_failure=restart_on_failure,
    )
    task.add_done_callback(callback)
    self.tasks.append(task)

build_log_context staticmethod #

build_log_context(
    message: Optional[StreamMessage[Any]], channel: str = ""
) -> dict[str, str]
Source code in faststream/redis/subscriber/usecases/basic.py
123
124
125
126
127
128
129
130
131
@staticmethod
def build_log_context(
    message: Optional["BrokerStreamMessage[Any]"],
    channel: str = "",
) -> dict[str, str]:
    return {
        "channel": channel,
        "message_id": getattr(message, "message_id", ""),
    }

consume_one async #

consume_one(msg: Any) -> None
Source code in faststream/redis/subscriber/usecases/basic.py
133
134
async def consume_one(self, msg: Any) -> None:
    await self.consume(msg)