Skip to content

Commit 3165457

Browse files
Ananto30Ananto30
committed
Update benchmarks
1 parent 755a3d0 commit 3165457

46 files changed

Lines changed: 754 additions & 242 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -366,15 +366,15 @@ Compare the results! 👇
366366

367367
| Framework | "hello world" (req/s) | 99% latency (ms) | redis save (req/s) | 99% latency (ms) |
368368
| ----------- | --------------------- | ---------------- | ------------------ | ---------------- |
369-
| aiohttp | 35023.07 | 7.66 | 17153.06 | 12.09 |
370-
| aiozmq | 20621.64 | 6.88 | 7651.61 | 11.13 |
371-
| blacksheep | 21887.95 | 8.9 | 11446.89 | 15.77 |
372-
| fastApi | 19857.4 | 9.66 | 9781.64 | 18.58 |
373-
| sanic | 46143.93 | 5.22 | 20062.11 | 9.77 |
374-
| zero(sync) | 22556.58 | 8.86 | 11779.06 | 20.13 |
375-
| zero(async) | 34411.23 | 5.73 | 23638.82 | 8.13 |
376-
377-
Seems like sanic is the fastest in hello world response, but zero is the fastest in redis save 🎉
369+
| aiohttp | 33167.69 | 11.89 | 17959.46 | 12.76 |
370+
| aiozmq | 25174.24 | 6.13 | 8850.15 | 10.19 |
371+
| blacksheep | 38025.53 | 8.41 | 16324.19 | 13.54 |
372+
| fastApi | 19682.99 | 9.09 | 12775.97 | 16.28 |
373+
| sanic | 58811.27 | 4.43 | 23622.69 | 9.22 |
374+
| zero(sync) | 27570.85 | 6.65 | 10269.1 | 23.71 |
375+
| zero(async) | 41091.96 | 4.41 | 23996.18 | 8.64 |
376+
| zero(tcp) | 100752.12 | 2.33 | 35812.88 | 13.48 |
377+
378378

379379
# Contribution
380380

‎benchmarks/benchmarks_md_table.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
"benchmark-fastapi",
99
"benchmark-sanic",
1010
"benchmark-zero",
11+
"benchmark-zero-tcp",
1112
]
1213

1314

1415
def get_latest_history():
1516
# file format is `<available_benchmark>__<date>.log`
16-
history_dir = "dockerize/history"
17+
history_dir = "benchmarks/dockerize/history"
1718
if not os.path.exists(history_dir):
1819
return []
1920
files = os.listdir(history_dir)
@@ -87,6 +88,7 @@ def make_comparison_table(all_results):
8788
"sanic": "sanic",
8889
"zero": "zero(sync)",
8990
"zero-async": "zero(async)",
91+
"zero-tcp": "zero(tcp)",
9092
}
9193
# Prepare a dict to collect results
9294
table_data = {fw: {} for fw in framework_map.values()}
@@ -102,6 +104,11 @@ def classify(framework, test):
102104
return "zero(async)", "hello"
103105
elif test == "async_order":
104106
return "zero(async)", "order"
107+
elif framework == "zero-tcp":
108+
if test == "async_hello":
109+
return "zero(tcp)", "hello"
110+
elif test == "async_order":
111+
return "zero(tcp)", "order"
105112
else:
106113
if test == "hello":
107114
return framework_map[framework], "hello"
@@ -129,6 +136,7 @@ def classify(framework, test):
129136
"sanic",
130137
"zero(sync)",
131138
"zero(async)",
139+
"zero(tcp)",
132140
]:
133141
hello = table_data[fw].get("hello", {})
134142
order = table_data[fw].get("order", {})
@@ -145,7 +153,7 @@ def classify(framework, test):
145153
history_files = get_latest_history()
146154
all_results = {}
147155
for history_file in history_files:
148-
filepath = os.path.join("dockerize/history", history_file)
156+
filepath = os.path.join("benchmarks/dockerize/history", history_file)
149157
if os.path.exists(filepath):
150158
benchmark_name = history_file.split("__")[0]
151159
results = parse_benchmark_log(filepath)

‎benchmarks/dockerize/Makefile‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,74 @@
11
SHELL := /bin/bash
22
benchmark-aiohttp:
33
cd aiohttp && ( \
4-
docker-compose up -d --build gateway server redis; \
4+
docker compose up -d --build gateway server redis; \
55
sleep 2; \
6-
docker-compose up wrk-hello >> ../history/benchmark-aiohttp__$(shell date +%Y-%m-%d).log 2>&1; \
7-
docker-compose up wrk-order >> ../history/benchmark-aiohttp__$(shell date +%Y-%m-%d).log 2>&1; \
8-
docker-compose down; \
6+
docker compose up wrk-hello >> ../history/benchmark-aiohttp__$(shell date +%Y-%m-%d).log 2>&1; \
7+
docker compose up wrk-order >> ../history/benchmark-aiohttp__$(shell date +%Y-%m-%d).log 2>&1; \
8+
docker compose down; \
99
)
1010

1111
benchmark-blacksheep:
1212
cd blacksheep && ( \
13-
docker-compose up -d --build gateway server redis; \
13+
docker compose up -d --build gateway server redis; \
1414
sleep 2; \
15-
docker-compose up wrk-hello >> ../history/benchmark-blacksheep__$(shell date +%Y-%m-%d).log 2>&1; \
16-
docker-compose up wrk-order >> ../history/benchmark-blacksheep__$(shell date +%Y-%m-%d).log 2>&1; \
17-
docker-compose down; \
15+
docker compose up wrk-hello >> ../history/benchmark-blacksheep__$(shell date +%Y-%m-%d).log 2>&1; \
16+
docker compose up wrk-order >> ../history/benchmark-blacksheep__$(shell date +%Y-%m-%d).log 2>&1; \
17+
docker compose down; \
1818
)
1919

2020
benchmark-aiozmq:
2121
cd aiozmq && ( \
22-
docker-compose up -d --build gateway server redis; \
22+
docker compose up -d --build gateway server redis; \
2323
sleep 2; \
24-
docker-compose up wrk-hello >> ../history/benchmark-aiozmq__$(shell date +%Y-%m-%d).log 2>&1; \
25-
docker-compose up wrk-order >> ../history/benchmark-aiozmq__$(shell date +%Y-%m-%d).log 2>&1; \
26-
docker-compose down; \
24+
docker compose up wrk-hello >> ../history/benchmark-aiozmq__$(shell date +%Y-%m-%d).log 2>&1; \
25+
docker compose up wrk-order >> ../history/benchmark-aiozmq__$(shell date +%Y-%m-%d).log 2>&1; \
26+
docker compose down; \
2727
)
2828

2929
benchmark-fastapi:
3030
cd fast_api && ( \
31-
docker-compose up -d --build gateway server redis; \
31+
docker compose up -d --build gateway server redis; \
3232
sleep 2; \
33-
docker-compose up wrk-hello >> ../history/benchmark-fastapi__$(shell date +%Y-%m-%d).log 2>&1; \
34-
docker-compose up wrk-order >> ../history/benchmark-fastapi__$(shell date +%Y-%m-%d).log 2>&1; \
35-
docker-compose down; \
33+
docker compose up wrk-hello >> ../history/benchmark-fastapi__$(shell date +%Y-%m-%d).log 2>&1; \
34+
docker compose up wrk-order >> ../history/benchmark-fastapi__$(shell date +%Y-%m-%d).log 2>&1; \
35+
docker compose down; \
3636
)
3737

3838
benchmark-sanic:
3939
cd sanic && ( \
40-
docker-compose up -d --build gateway server redis; \
40+
docker compose up -d --build gateway server redis; \
4141
sleep 2; \
42-
docker-compose up wrk-hello >> ../history/benchmark-sanic__$(shell date +%Y-%m-%d).log 2>&1; \
43-
docker-compose up wrk-order >> ../history/benchmark-sanic__$(shell date +%Y-%m-%d).log 2>&1; \
44-
docker-compose down; \
42+
docker compose up wrk-hello >> ../history/benchmark-sanic__$(shell date +%Y-%m-%d).log 2>&1; \
43+
docker compose up wrk-order >> ../history/benchmark-sanic__$(shell date +%Y-%m-%d).log 2>&1; \
44+
docker compose down; \
4545
)
4646

4747
benchmark-zero:
4848
cd zero && ( \
49-
docker-compose up -d --build gateway server redis; \
49+
docker compose up -d --build gateway server redis; \
5050
sleep 2; \
51-
docker-compose up wrk-hello >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
52-
docker-compose up wrk-order >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
53-
docker-compose up wrk-hello-async >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
54-
docker-compose up wrk-order-async >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
51+
docker compose up wrk-hello >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
52+
docker compose up wrk-order >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
53+
docker compose up wrk-hello-async >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
54+
docker compose up wrk-order-async >> ../history/benchmark-zero__$(shell date +%Y-%m-%d).log 2>&1; \
5555
docker-compose down; \
5656
)
5757

58+
benchmark-zero-tcp:
59+
cd zero_tcp && ( \
60+
docker compose up -d --build gateway server redis; \
61+
sleep 2; \
62+
docker compose up wrk-hello-async >> ../history/benchmark-zero-tcp__$(shell date +%Y-%m-%d).log 2>&1; \
63+
docker compose up wrk-order-async >> ../history/benchmark-zero-tcp__$(shell date +%Y-%m-%d).log 2>&1; \
64+
docker compose down; \
65+
)
66+
5867
benchmark-all:
5968
$(MAKE) benchmark-aiohttp
6069
$(MAKE) benchmark-blacksheep
6170
$(MAKE) benchmark-aiozmq
6271
$(MAKE) benchmark-fastapi
6372
$(MAKE) benchmark-sanic
64-
$(MAKE) benchmark-zero
73+
$(MAKE) benchmark-zero
74+
$(MAKE) benchmark-zero-tcp

‎benchmarks/dockerize/aiohttp/docker-compose.yaml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,20 @@
1-
version: '3'
2-
31
services:
42
gateway:
53
build: .
64
container_name: gateway
75
command: gunicorn gateway:app --bind 0.0.0.0:8000 --worker-class aiohttp.worker.GunicornWebWorker --workers 8 --log-level warning
8-
ports:
9-
- "8000:8000"
106
depends_on:
117
- server
128
- redis
139
server:
1410
build: .
1511
container_name: server
1612
command: gunicorn server:app --bind 0.0.0.0:8011 --worker-class aiohttp.worker.GunicornWebWorker --workers 8 --log-level warning
17-
ports:
18-
- "8011:8011"
1913
# cpus: '0.50'
2014
# mem_limit: 256m
2115
redis:
2216
image: eqalpha/keydb:latest
2317
container_name: redis
24-
ports:
25-
- "6379:6379"
2618
wrk-hello:
2719
image: skandyla/wrk
2820
command: -t 8 -c 80 -d 30s --latency http://gateway:8000/hello

‎benchmarks/dockerize/aiohttp/gateway.py‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import logging
22
from typing import Optional
33

4-
from aiohttp import ClientSession, web
4+
import ujson
5+
from aiohttp import ClientSession, TCPConnector, web
56

67
logger = logging.getLogger(__name__)
78

@@ -10,27 +11,40 @@
1011

1112
uvloop.install()
1213
except ImportError:
13-
logger.warn("Cannot use uvloop")
14-
pass
14+
logger.warning("Cannot use uvloop")
1515

1616
session: Optional[ClientSession] = None
1717

1818

19-
async def hello(request):
19+
async def init_session(app):
20+
"""Initialize session at app startup"""
2021
global session
21-
if session is None:
22-
session = ClientSession()
22+
connector = TCPConnector(
23+
limit=100, # Total connections
24+
limit_per_host=30, # Connections per host
25+
ttl_dns_cache=300, # DNS cache TTL
26+
keepalive_timeout=30, # Keep-alive timeout
27+
)
28+
session = ClientSession(
29+
connector=connector,
30+
json_serialize=ujson.dumps, # Faster JSON
31+
cookie_jar=None, # Disable cookies if not needed
32+
)
33+
app["session"] = session
34+
35+
36+
async def cleanup_session(app):
37+
"""Clean up session on shutdown"""
38+
await session.close()
2339

40+
41+
async def hello(request):
2442
resp = await session.get("http://server:8011/hello")
2543
txt = await resp.text()
2644
return web.Response(text=txt)
2745

2846

2947
async def order(request):
30-
global session
31-
if session is None:
32-
session = ClientSession()
33-
3448
resp = await session.post(
3549
"http://server:8011/order",
3650
json={"user_id": "1", "items": ["apple", "python"]},
@@ -39,5 +53,7 @@ async def order(request):
3953

4054

4155
app = web.Application()
56+
app.on_startup.append(init_session)
57+
app.on_cleanup.append(cleanup_session)
4258
app.router.add_get("/order", order)
4359
app.router.add_get("/hello", hello)

‎benchmarks/dockerize/aiohttp/requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ gunicorn
33
PyJWT
44
redis>=4.2.0rc1
55
uvloop
6-
msgpack
6+
msgpack
7+
ujson

‎benchmarks/dockerize/aiohttp/server.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@
1212

1313
uvloop.install()
1414
except ImportError:
15-
logger.warn("Cannot use uvloop")
16-
pass
15+
logger.warning("Cannot use uvloop")
1716

1817

1918
async def hello(request):

‎benchmarks/dockerize/aiohttp/shared.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,6 @@ class OrderStatus:
3232

3333

3434
class Order(Btype):
35-
# cache
36-
orders = {}
37-
3835
def __init__(self, id, items, created_by, created_at, status, updated_at=None):
3936
self.id = id
4037
self.items = items
@@ -89,15 +86,6 @@ async def async_get_order(id: str) -> Order:
8986
return Order.unpack(await r.get(id))
9087

9188

92-
class SingletonMeta(type):
93-
_instance = None
94-
95-
def __call__(self):
96-
if self._instance is None:
97-
self._instance = super().__call__()
98-
return self._instance
99-
100-
10189
class RedisClient(metaclass=SingletonMeta):
10290
def __init__(self):
10391
self.host = "0.0.0.0"

‎benchmarks/dockerize/aiozmq/docker-compose.yaml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,20 @@
1-
version: '3'
2-
31
services:
42
gateway:
53
build: .
64
container_name: gateway
75
command: gunicorn gateway:app --bind 0.0.0.0:8000 --worker-class aiohttp.worker.GunicornWebWorker --workers 8 --log-level warning
8-
ports:
9-
- "8000:8000"
106
depends_on:
117
- server
128
- redis
139
server:
1410
build: .
1511
container_name: server
1612
command: python server.py
17-
ports:
18-
- "5559:5559"
1913
# cpus: '0.40'
2014
# mem_limit: 256m
2115
redis:
2216
image: eqalpha/keydb:latest
2317
container_name: redis
24-
ports:
25-
- "6379:6379"
2618
wrk-hello:
2719
image: skandyla/wrk
2820
command: -t 8 -c 80 -d 30s --latency http://gateway:8000/hello

‎benchmarks/dockerize/aiozmq/gateway.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
uvloop.install()
1010
except ImportError:
1111
logging.warning("Cannot use uvloop")
12-
pass
1312

1413
aiozmq_client = None
1514

0 commit comments

Comments
 (0)