Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/02_concepts/05_retries.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import ApiLink from '@theme/ApiLink';

import RetriesAsyncExample from '!!raw-loader!./code/05_retries_async.py';
import RetriesSyncExample from '!!raw-loader!./code/05_retries_sync.py';
import StatsAsyncExample from '!!raw-loader!./code/05_stats_async.py';
import StatsSyncExample from '!!raw-loader!./code/05_stats_sync.py';
import WaitForResourcesAsyncExample from '!!raw-loader!./code/05_wait_for_resources_async.py';
import WaitForResourcesSyncExample from '!!raw-loader!./code/05_wait_for_resources_sync.py';

Expand Down Expand Up @@ -46,6 +48,29 @@ Retries with exponential backoff help reduce the load on the server and increase
</TabItem>
</Tabs>

## Request statistics

The client counts its calls, HTTP requests, and rate limit errors. To read the counts, use the <ApiLink to="class/ApifyClient#stats">`stats`</ApiLink> property. It returns a <ApiLink to="class/ClientStatistics">`ClientStatistics`</ApiLink> object with these fields:

- `calls` is the number of calls made through the client, including requests to Actor servers.
- `requests` is the number of HTTP requests sent, including retries and requests to Actor servers.
- `rate_limit_errors` maps each attempt to the number of rate limit (429) errors it got. Key `0` is the first attempt, key `1` is the first retry, and so on. The field is a `defaultdict`, so reading a missing key adds it. To read a count without changing the dict, use `.get()`. Only API calls count here. A 429 from an Actor server called through <ApiLink to="class/ActorServerClient#request">`ActorServerClient.request`</ApiLink> is not counted.

With a custom HTTP client, `stats` returns the statistics of that client. For details, see [HTTP clients](/api/client/python/docs/concepts/custom-http-clients).

<Tabs>
<TabItem value="AsyncExample" label="Async client" default>
<CodeBlock className="language-python">
{StatsAsyncExample}
</CodeBlock>
</TabItem>
<TabItem value="SyncExample" label="Sync client">
<CodeBlock className="language-python">
{StatsSyncExample}
</CodeBlock>
</TabItem>
</Tabs>

## Wait for resources to start a run

Starting a run fails with an HTTP 402 error when the account doesn't have enough free memory for the run, or when it already runs as many Actors as its plan allows. The error `type` is `actor-memory-limit-exceeded` or `concurrent-runs-limit-exceeded`. The client doesn't retry these errors on its own, since they clear only after other runs or builds of the account finish.
Expand Down
13 changes: 13 additions & 0 deletions docs/02_concepts/code/05_stats_async.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from apify_client import ApifyClientAsync

TOKEN = 'MY-APIFY-TOKEN'


async def main() -> None:
apify_client = ApifyClientAsync(TOKEN)

await apify_client.actor('username/actor-name').call()

stats = apify_client.stats
print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}')
print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}')
13 changes: 13 additions & 0 deletions docs/02_concepts/code/05_stats_sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from apify_client import ApifyClient

TOKEN = 'MY-APIFY-TOKEN'


def main() -> None:
apify_client = ApifyClient(TOKEN)

apify_client.actor('username/actor-name').call()

stats = apify_client.stats
print(f'API calls: {stats.calls}, HTTP requests: {stats.requests}')
print(f'Rate limit errors per attempt: {dict(stats.rate_limit_errors)}')
2 changes: 2 additions & 0 deletions src/apify_client/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from importlib import metadata

from ._apify_client import ApifyClient, ApifyClientAsync
from ._statistics import ClientStatistics

__version__ = metadata.version('apify-client')

__all__ = [
'ApifyClient',
'ApifyClientAsync',
'ClientStatistics',
'__version__',
]
28 changes: 19 additions & 9 deletions src/apify_client/_apify_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,14 @@
WebhookDispatchCollectionClient,
WebhookDispatchCollectionClientAsync,
)
from apify_client._statistics import ClientStatistics
from apify_client._utils.http import check_custom_headers, to_api_base_url
from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
from apify_client.http_compressors._resolve import resolve_compressor

if TYPE_CHECKING:
from datetime import timedelta

from apify_client._statistics import ClientStatistics
from apify_client.http_compressors._base import HttpCompressor
from apify_client.types import HttpCompressionAlgorithm

Expand Down Expand Up @@ -182,9 +182,6 @@ def __init__(
self._public_base_url = to_api_base_url(api_public_url)
"""Public base URL for CDN access."""

self._statistics = ClientStatistics()
"""Collector for client request statistics."""

self._http_client: HttpClient | None = None
"""HTTP client used to communicate with the Apify API. Lazily initialized on first access."""

Expand Down Expand Up @@ -284,6 +281,15 @@ def token(self) -> str | None:
"""The Apify API token used by the client."""
return self._token

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors.

These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only
what its own implementation records.
"""
return self.http_client.stats

@property
def http_client(self) -> HttpClient:
"""The HTTP client instance used for API communication.
Expand All @@ -300,7 +306,6 @@ def http_client(self) -> HttpClient:
timeout_max=self._timeout_max,
max_retries=self._max_retries,
min_delay_between_retries=self._min_delay_between_retries,
statistics=self._statistics,
headers=self._headers,
user_agent_suffix=self._user_agent_suffix,
http_compressor=self._http_compressor,
Expand Down Expand Up @@ -565,9 +570,6 @@ def __init__(
self._public_base_url = to_api_base_url(api_public_url)
"""Public base URL for CDN access."""

self._statistics = ClientStatistics()
"""Collector for client request statistics."""

self._http_client: HttpClientAsync | None = None
"""HTTP client used to communicate with the Apify API. Lazily initialized on first access."""

Expand Down Expand Up @@ -667,6 +669,15 @@ def token(self) -> str | None:
"""The Apify API token used by the client."""
return self._token

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors.

These are the statistics of the HTTP client in use. A custom HTTP client that overrides `call` counts only
what its own implementation records.
"""
return self.http_client.stats

@property
def http_client(self) -> HttpClientAsync:
"""The HTTP client instance used for API communication.
Expand All @@ -683,7 +694,6 @@ def http_client(self) -> HttpClientAsync:
timeout_max=self._timeout_max,
max_retries=self._max_retries,
min_delay_between_retries=self._min_delay_between_retries,
statistics=self._statistics,
headers=self._headers,
user_agent_suffix=self._user_agent_suffix,
http_compressor=self._http_compressor,
Expand Down
10 changes: 8 additions & 2 deletions src/apify_client/_statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,25 @@
from collections import defaultdict
from dataclasses import dataclass, field

from apify_client._docs import docs_group


@docs_group('Other')
@dataclass
class ClientStatistics:
"""Statistics about API client usage and rate limit errors."""

calls: int = 0
"""Total number of API method calls made by the client."""
"""Total number of calls made by the client, including requests to Actor servers."""

requests: int = 0
"""Total number of HTTP requests sent, including retries."""

rate_limit_errors: defaultdict[int, int] = field(default_factory=lambda: defaultdict(int))
"""List tracking which retry attempts encountered rate limit (429) errors."""
"""Number of rate limit (429) errors per attempt, keyed by the 0-based attempt index.

Reading a missing key adds it with a count of `0`. Use `.get()` to read a count without changing the dict.
"""

def add_rate_limit_error(self, attempt: int) -> None:
"""Add rate limit error for specific attempt.
Expand Down
5 changes: 5 additions & 0 deletions src/apify_client/http_clients/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,11 @@ def __init__(

self._headers = merge_headers(default_headers, headers)

@property
def stats(self) -> ClientStatistics:
"""Statistics about API client usage and rate limit errors."""
return self._statistics

def set_default_authorization(self, token: str) -> None:
"""Set the `Authorization` header from the token, unless an authorization header is already configured.

Expand Down
2 changes: 1 addition & 1 deletion tests/unit/test_actor_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ def test_request_rate_limit_is_left_out_of_statistics(httpserver: HTTPServer) ->

client.actor(_MOCKED_ACTOR_ID).server().request('/flaky')

assert client._statistics.rate_limit_errors == {}
assert client.stats.rate_limit_errors == {}


def test_request_returns_last_retried_response(httpserver: HTTPServer) -> None:
Expand Down
3 changes: 1 addition & 2 deletions tests/unit/test_http_clients.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
import impit
import pytest

from apify_client import ApifyClient, ApifyClientAsync
from apify_client import ApifyClient, ApifyClientAsync, ClientStatistics
from apify_client._consts import MIN_COMPRESSION_SIZE
from apify_client._logging import logger_name
from apify_client._statistics import ClientStatistics
from apify_client.errors import ApifyApiError, InvalidResponseBodyError
from apify_client.http_clients import (
HttpClient,
Expand Down
109 changes: 108 additions & 1 deletion tests/unit/test_statistics.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,25 @@
from __future__ import annotations

from datetime import timedelta
from typing import TYPE_CHECKING

import pytest
from werkzeug import Response

import apify_client
from apify_client import ApifyClient, ApifyClientAsync, ClientStatistics
from apify_client.http_clients import ImpitHttpClient, ImpitHttpClientAsync

if TYPE_CHECKING:
from pytest_httpserver import HTTPServer


from apify_client._statistics import ClientStatistics
class _CustomHttpClient(ImpitHttpClient):
pass


class _CustomHttpClientAsync(ImpitHttpClientAsync):
pass


@pytest.mark.parametrize(
Expand Down Expand Up @@ -120,3 +137,93 @@ def test_add_rate_limit_error_large_attempt() -> None:

stats.add_rate_limit_error(10000)
assert stats.rate_limit_errors[9999] == 1


def test_client_statistics_is_exported() -> None:
"""Test that ClientStatistics is exported from the top-level apify_client package."""
assert 'ClientStatistics' in apify_client.__all__
assert apify_client.ClientStatistics is ClientStatistics


def test_apify_client_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClient.stats tracks calls and requests on the default HTTP client."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
client = ApifyClient('test-token', api_url=httpserver.url_for('/'))

assert isinstance(client.stats, ClientStatistics)
assert client.stats is client.http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1


def test_apify_client_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClient.with_custom_http_client tracks rate limits on a custom HTTP client subclass."""
httpserver.expect_oneshot_request('/v2/logs/test-id').respond_with_response(Response(status=429))
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
custom_http_client = _CustomHttpClient(min_delay_between_retries=timedelta(milliseconds=1))
client = ApifyClient.with_custom_http_client(
'test-token',
api_url=httpserver.url_for('/'),
http_client=custom_http_client,
)

assert client.stats is custom_http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 2
assert client.stats.rate_limit_errors == {0: 1}


async def test_apify_client_async_exposes_default_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClientAsync.stats tracks calls and requests on the default HTTP client."""
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
client = ApifyClientAsync('test-token', api_url=httpserver.url_for('/'))

assert isinstance(client.stats, ClientStatistics)
assert client.stats is client.http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

await client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 1


async def test_apify_client_async_exposes_custom_http_client_statistics(httpserver: HTTPServer) -> None:
"""Test that ApifyClientAsync.with_custom_http_client tracks rate limits on a custom HTTP client subclass."""
httpserver.expect_oneshot_request('/v2/logs/test-id').respond_with_response(Response(status=429))
httpserver.expect_request('/v2/logs/test-id').respond_with_response(
Response('ok', status=200, content_type='text/plain')
)
custom_http_client = _CustomHttpClientAsync(min_delay_between_retries=timedelta(milliseconds=1))
client = ApifyClientAsync.with_custom_http_client(
'test-token',
api_url=httpserver.url_for('/'),
http_client=custom_http_client,
)

assert client.stats is custom_http_client.stats
assert client.stats.calls == 0
assert client.stats.requests == 0

await client.log('test-id').get()

assert client.stats.calls == 1
assert client.stats.requests == 2
assert client.stats.rate_limit_errors == {0: 1}
Loading