Skip to content

Commit 87ef56e

Browse files
Mark-HMark-H
committed
First stab at implementing asynchronously pushed-back task results
The general idea is to not keep workers waiting for tasks that may take a while to complete. Things like backups and upgrades may take the client a minute to process - the worker does not need to sit idle during that time; it might as well process other tasks simultaniously. Flow: 1. Worker pushes the task to the client, with a uri and one-time signing key 2. Client acknowledges receipt and support for the async return push with a 202 3. Worker marks task as awaiting async response and continues to whatever other tasks are queued 4. Client processes slow task, and when done pushes the response to the sitedash endpoint from (1). Endpoint stores received data, marks related job exec as having received data to process. 5. Same worker (when free) is assigned again to job exec and finishes processing the data
1 parent 935801f commit 87ef56e

File tree

5 files changed

+174
-18
lines changed

5 files changed

+174
-18
lines changed

‎assets/components/sitedashclient/pull.php‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@
4545
// Make sure the params are sanitized
4646
$params = $modx::sanitize($_POST);
4747

48+
$pusher = null;
49+
if (array_key_exists('_return_push', $_POST) && array_key_exists('_return_push_key', $_POST)) {
50+
$server = $modx->getOption('sitedash.server_uri', null, 'https://sitedash.app/', true);
51+
$responseUri = (string)$_POST['_return_push'];
52+
$signingKey = (string)$_POST['_return_push_key'];
53+
54+
$pusher = new \modmore\SiteDashClient\Communication\Pusher($server, $responseUri, $signingKey);
55+
}
56+
4857
switch ($params['request']) {
4958
case 'system':
5059
case 'system/refresh':
@@ -91,7 +100,7 @@
91100

92101

93102
case 'upgrade/backup':
94-
$cmd = new \modmore\SiteDashClient\Upgrade\Backup($modx);
103+
$cmd = new \modmore\SiteDashClient\Upgrade\Backup($modx, $pusher);
95104
$cmd->run();
96105
break;
97106

‎core/components/sitedashclient/src/Communication/Pusher.php‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?php
2+
3+
namespace modmore\SiteDashClient\Communication;
4+
5+
final class Pusher {
6+
private $responseUri;
7+
private $signingKey;
8+
9+
public function __construct($server, $responseUri, $signingKey)
10+
{
11+
$this->responseUri = $server . $responseUri;
12+
$this->signingKey = base64_decode($signingKey);
13+
}
14+
15+
public function acknowledge()
16+
{
17+
ob_start();
18+
19+
echo json_encode([
20+
'return_push' => true,
21+
]);
22+
23+
// Get the size of the output.
24+
$size = ob_get_length();
25+
26+
// 202 accepted
27+
http_response_code(202);
28+
29+
// Disable compression (in case content length is compressed).
30+
header('Content-Encoding: none');
31+
32+
// Set the content length of the response.
33+
header("Content-Length: {$size}");
34+
35+
// Close the connection.
36+
header('Connection: close');
37+
38+
// Flush all output.
39+
ob_end_flush();
40+
ob_flush();
41+
flush();
42+
43+
ignore_user_abort(true);
44+
@session_write_close();
45+
46+
if (is_callable('fastcgi_finish_request')) {
47+
fastcgi_finish_request();
48+
return;
49+
}
50+
sleep(1);
51+
}
52+
53+
public function push(array $data)
54+
{
55+
$logFile = MODX_CORE_PATH . 'cache/logs/sitedash_push_' . date('Y-m-d-H-i-s') . '.log';
56+
57+
$ch = curl_init();
58+
59+
$postData = $this->prepareData($data);
60+
curl_setopt($ch, CURLOPT_URL, $this->responseUri);
61+
curl_setopt($ch, CURLOPT_POST, true);
62+
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($postData));
63+
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
64+
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type:application/json']);
65+
66+
$response = curl_exec($ch);
67+
$error = curl_error($ch);
68+
$errno = curl_errno($ch);
69+
curl_close($ch);
70+
71+
$dataFormat = json_encode($data, JSON_PRETTY_PRINT);
72+
$postDataFormat = json_encode($postData, JSON_PRETTY_PRINT);
73+
$log = <<<HTML
74+
Push requested to {$this->responseUri} with one-time use signing key:
75+
76+
{$this->signingKey}
77+
78+
Data: {$dataFormat}
79+
80+
Data to post to SiteDash, incl signature: {$postDataFormat}
81+
82+
Response from SiteDash: {$errno} {$error}
83+
84+
{$response}
85+
HTML;
86+
87+
file_put_contents($logFile, $log);
88+
}
89+
90+
private function prepareData(array $data)
91+
{
92+
return [
93+
'data' => $data,
94+
'signature' => $this->sign($data),
95+
];
96+
}
97+
98+
private function sign(array $data)
99+
{
100+
$sigData = json_encode($data);
101+
102+
$binary_signature = '';
103+
openssl_sign($sigData, $binary_signature, $this->signingKey, OPENSSL_ALGO_SHA1);
104+
105+
// Encode it as base64
106+
$binary_signature = base64_encode($binary_signature);
107+
return $binary_signature;
108+
}
109+
}

‎core/components/sitedashclient/src/Communication/Result.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
namespace modmore\SiteDashClient\Communication;
4+
5+
final class Result {
6+
/**
7+
* @var Pusher|null
8+
*/
9+
private $pusher;
10+
11+
public function __construct(Pusher $pusher = null)
12+
{
13+
$this->pusher = $pusher;
14+
}
15+
16+
public function __invoke($responseCode, $data)
17+
{
18+
if ($this->pusher) {
19+
$this->pusher->push($data);
20+
}
21+
else {
22+
http_response_code($responseCode);
23+
echo json_encode($data, JSON_PRETTY_PRINT);
24+
@session_write_close();
25+
exit();
26+
}
27+
}
28+
}

‎core/components/sitedashclient/src/Refresh.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public function run()
1717
$data = [];
1818
$data['client'] = \SiteDashClient::VERSION;
1919
$data['client_options'] = [
20-
'supports_return_push' => false,
20+
'supports_return_push' => true,
2121
'supports_async_execute' => false,
2222
];
2323
$data['modx'] = $this->getMODXData();

‎core/components/sitedashclient/src/Upgrade/Backup.php‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,24 @@
33
namespace modmore\SiteDashClient\Upgrade;
44

55
use modmore\SiteDashClient\CommandInterface;
6+
use modmore\SiteDashClient\Communication\Pusher;
7+
use modmore\SiteDashClient\Communication\Result;
68
use Symfony\Component\Process\ExecutableFinder;
79
use Symfony\Component\Process\Process;
810

911
class Backup implements CommandInterface {
1012
protected $modx;
1113
protected $files = [];
1214
protected $targetDirectory;
15+
/**
16+
* @var Pusher|null
17+
*/
18+
private $pusher;
1319

14-
public function __construct(\modX $modx)
20+
public function __construct(\modX $modx, $pusher = null)
1521
{
1622
$this->modx = $modx;
23+
$this->pusher = $pusher;
1724

1825
$this->files = [
1926
MODX_CORE_PATH . 'config/' . MODX_CONFIG_KEY . '.inc.php',
@@ -66,6 +73,13 @@ public function run()
6673
return;
6774
}
6875

76+
// If a push result was requested, send an ack response and continue processing
77+
if ($this->pusher) {
78+
$this->pusher->acknowledge();
79+
}
80+
81+
$result = new Result($this->pusher);
82+
6983
/**
7084
* Include the config file to access the database information
7185
*
@@ -112,51 +126,48 @@ public function run()
112126
$msg = str_replace($password_parameter, '-p\'<PASS>\'', $msg);
113127
$trace = $e->getTraceAsString();
114128
$trace = str_replace($password_parameter, '-p\'<PASS>\'', $trace);
115-
http_response_code(503);
116-
echo json_encode([
129+
130+
$result(503, [
117131
'success' => false,
118132
'message' => 'Received an error trying to run mysqlbackup: ' . $msg,
119133
'binary' => $mysqldump,
120134
'directory' => str_replace(MODX_CORE_PATH, '{core_path}', $this->targetDirectory),
121135
'output' => $trace,
122-
], JSON_PRETTY_PRINT);
136+
]);
123137
return;
124138
}
125139
$output = $backupProcess->getErrorOutput() . ' ' . $backupProcess->getOutput();
126140
$output = str_replace($password_parameter, '-p\'<PASS>\'', $output);
127141
if (!$backupProcess->isSuccessful()) {
128-
http_response_code(503);
129142
$code = $backupProcess->getExitCode();
130143
if ($code === 127) {
131-
echo json_encode([
144+
$result(503, [
132145
'success' => false,
133146
'message' => 'Could not find the mysqldump program on your server; please configure the sitedashclient.mysqldump_binary system setting to point to mysqldump to create backups.',
134147
'binary' => $mysqldump,
135148
'directory' => str_replace(MODX_CORE_PATH, '{core_path}', $this->targetDirectory),
136149
'output' => $output,
137-
], JSON_PRETTY_PRINT);
150+
]);
138151
return;
139152
}
140153

141-
echo json_encode([
154+
$result(503, [
142155
'success' => false,
143156
'message' => 'Received exit code ' . $code . ' trying to create a database backup using ' . $mysqldump . ' with message: ' . $output,
144157
'output' => $output,
145158
'return' => $code,
146-
], JSON_PRETTY_PRINT);
159+
]);
147160
return;
148161
}
149162

150163
$backupSize = filesize($targetFile);
151164
if ($backupSize < 150 * 1024) { // a clean install is ~ 200kb, so we ask for at least 150
152-
http_response_code(503);
153-
154-
echo json_encode([
165+
$result(503, [
155166
'success' => false,
156167
'message' => 'While the backup with ' . $mysqldump . ' did not indicate an error, the mysql backup is only ' . number_format($backupSize / 1024, 0) . 'kb in size, so it probably failed.',
157168
'output' => $output,
158169
'return' => $backupProcess->getExitCode(),
159-
], JSON_PRETTY_PRINT);
170+
]);
160171
return;
161172
}
162173

@@ -179,11 +190,10 @@ public function run()
179190
}
180191
}
181192

182-
http_response_code(200);
183-
echo json_encode([
193+
$result(200, [
184194
'success' => true,
185195
'directory' => str_replace(MODX_CORE_PATH, '', $this->targetDirectory),
186-
], JSON_PRETTY_PRINT);
196+
]);
187197
}
188198

189199
private function createDirectory($target)

0 commit comments

Comments
 (0)