Skip to content

Commit ae0a074

Browse files
albanmalbanm
committed
refactor: extract axios related functions
1 parent a696b00 commit ae0a074

File tree

3 files changed

+112
-108
lines changed

3 files changed

+112
-108
lines changed

‎ui/dts/typed-router.d.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ declare module 'vue-router/auto-routes' {
2020
export interface RouteNamedMap {
2121
'/admin/': RouteRecordInfo<'/admin/', '/admin', Record<never, never>, Record<never, never>>,
2222
'/admin/plugins': RouteRecordInfo<'/admin/plugins', '/admin/plugins', Record<never, never>, Record<never, never>>,
23-
'/dev': RouteRecordInfo<'/dev', '/dev', Record<never, never>, Record<never, never>>,
2423
'/processings/': RouteRecordInfo<'/processings/', '/processings', Record<never, never>, Record<never, never>>,
2524
'/processings/[id]': RouteRecordInfo<'/processings/[id]', '/processings/:id', { id: ParamValue<true> }, { id: ParamValue<false> }>,
2625
'/processings/new': RouteRecordInfo<'/processings/new', '/processings/new', Record<never, never>, Record<never, never>>,

‎worker/src/task/axios.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { httpAgent, httpsAgent } from '@data-fair/lib-node/http-agents.js'
2+
import axios from 'axios'
3+
import axiosRetry from 'axios-retry'
4+
import config from '#config'
5+
import type { Processing } from '#api/types'
6+
import type { LogFunctions } from '@data-fair/lib-common-types/processings.js'
7+
8+
/**
9+
* Create an Axios instance.
10+
*/
11+
export const getAxiosInstance = (processing: Processing, log: LogFunctions) => {
12+
const headers: Record<string, string> = {
13+
'x-apiKey': config.dataFairAPIKey
14+
}
15+
if (config.dataFairAdminMode) {
16+
const account = { ...processing.owner }
17+
if (account.name) account.name = encodeURIComponent(account.name)
18+
if (account.departmentName) account.departmentName = encodeURIComponent(account.departmentName)
19+
headers['x-account'] = JSON.stringify(account)
20+
}
21+
headers['x-processing'] = JSON.stringify({ _id: processing._id, title: encodeURIComponent(processing.title) })
22+
23+
const axiosInstance = axios.create({
24+
// this is necessary to prevent excessive memory usage during large file uploads, see https://github.com/axios/axios/issues/1045
25+
maxRedirects: 0,
26+
httpAgent,
27+
httpsAgent
28+
})
29+
30+
// apply default base url and send api key when relevant
31+
axiosInstance.interceptors.request.use(cfg => {
32+
if (!cfg.url) throw new Error('missing url in axios request')
33+
if (!/^https?:\/\//i.test(cfg.url)) {
34+
if (cfg.url.startsWith('/')) cfg.url = config.dataFairUrl + cfg.url
35+
else cfg.url = config.dataFairUrl + '/' + cfg.url
36+
}
37+
const isDataFairUrl = cfg.url.startsWith(config.dataFairUrl)
38+
if (isDataFairUrl) Object.assign(cfg.headers, headers)
39+
cfg.headers['User-Agent'] = cfg.headers['User-Agent'] ?? `@data-fair/processings (${processing.plugin})`
40+
41+
// use private data fair url if specified to prevent leaving internal infrastructure
42+
// except from GET requests so that they still appear in metrics
43+
// except if config.getFromPrivateDataFairUrl is set to true, then all requests are sent to the private url
44+
const usePrivate =
45+
config.privateDataFairUrl &&
46+
isDataFairUrl &&
47+
(config.getFromPrivateDataFairUrl || ['post', 'put', 'delete', 'patch'].includes(cfg.method || ''))
48+
if (usePrivate) {
49+
cfg.url = cfg.url.replace(config.dataFairUrl, config.privateDataFairUrl!)
50+
cfg.headers.host = new URL(config.dataFairUrl).host
51+
}
52+
return cfg
53+
}, error => Promise.reject(error))
54+
55+
axiosRetry(axiosInstance, {
56+
retries: 3,
57+
retryDelay: axiosRetry.exponentialDelay,
58+
shouldResetTimeout: true,
59+
onRetry: (retryCount, _err, requestConfig) => {
60+
const err = prepareAxiosError(_err)
61+
const message = getHttpErrorMessage(err) || err.message || err
62+
log.warning(`tentative ${retryCount} de requête ${requestConfig.method} ${requestConfig.url} : ${message}`)
63+
}
64+
})
65+
66+
return axiosInstance
67+
}
68+
69+
// customize axios errors for shorter stack traces when a request fails
70+
// WARNING: we used to do it in an interceptor, but it was incompatible with axios-retry
71+
export const prepareAxiosError = (error: any) => {
72+
const response = error.response ?? error.request?.res ?? error.res
73+
if (!response) return error
74+
delete response.request
75+
const headers: Record<string, string> = {}
76+
if (response.headers?.location) headers.location = response.headers.location
77+
response.headers = headers
78+
response.config = response.config ?? error.config
79+
if (response.config) {
80+
response.config = { method: response.config.method, url: response.config.url, params: response.config.params, data: response.config.data }
81+
if (response.config.data && response.config.data._writableState) delete response.config.data
82+
}
83+
if (response.data && response.data._readableState) delete response.data
84+
if (error.message) response.message = error.message
85+
if (error.stack) response.stack = error.stack
86+
return response
87+
}
88+
89+
export const getHttpErrorMessage = (err: any) => {
90+
let httpMessage = err.status ?? err.statusCode
91+
if (httpMessage) {
92+
const statusText = err.statusText ?? err.statusMessage
93+
if (statusText) httpMessage += ' - ' + statusText
94+
if (err.data) {
95+
if (typeof err.data === 'string') httpMessage += ' - ' + err.data
96+
else httpMessage += ' - ' + JSON.stringify(err.data)
97+
} else if (err.message) {
98+
httpMessage += ' - ' + err.message
99+
}
100+
if (err.config && err.config.url) {
101+
let url = err.config.url
102+
url = url.replace(config.dataFairUrl, '')
103+
if (config.privateDataFairUrl) {
104+
url = url.replace(config.privateDataFairUrl, '')
105+
}
106+
httpMessage += ` (${url})`
107+
}
108+
return httpMessage
109+
}
110+
}

‎worker/src/task/task.ts‎

Lines changed: 2 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,18 @@ import type { LogFunctions, ProcessingContext } from '@data-fair/lib-common-type
22
import type { Account } from '@data-fair/lib-express/index.js'
33
import type { Processing, Run } from '#api/types'
44

5-
import axios from 'axios'
6-
import axiosRetry from 'axios-retry'
75
import util from 'node:util'
86
import fs from 'fs-extra'
97
import path from 'path'
108
import resolvePath from 'resolve-path'
119
import tmp from 'tmp-promise'
1210
import { DataFairWsClient } from '@data-fair/lib-node/ws-client.js'
13-
import { httpAgent, httpsAgent } from '@data-fair/lib-node/http-agents.js'
1411
import * as wsEmitter from '@data-fair/lib-node/ws-emitter.js'
1512
import { decipher } from '@data-fair/processings-shared/cipher.ts'
1613
import { running } from '../utils/runs.ts'
1714
import config from '#config'
1815
import mongo from '#mongo'
16+
import { getAxiosInstance, getHttpErrorMessage, prepareAxiosError } from './axios.ts'
1917

2018
fs.ensureDirSync(config.dataDir)
2119
const baseTmpDir = config.tmpDir || path.join(config.dataDir, 'tmp')
@@ -27,76 +25,6 @@ let pluginModule: { run: (context: ProcessingContext) => Promise<void>, stop?: (
2725
let _stopped: boolean
2826
const processingsDir = path.join(config.dataDir, 'processings')
2927

30-
/**
31-
* Create an Axios instance.
32-
*/
33-
const getAxiosInstance = (processing: Processing) => {
34-
const headers: Record<string, string> = {
35-
'x-apiKey': config.dataFairAPIKey
36-
}
37-
if (config.dataFairAdminMode) {
38-
const account = { ...processing.owner }
39-
if (account.name) account.name = encodeURIComponent(account.name)
40-
if (account.departmentName) account.departmentName = encodeURIComponent(account.departmentName)
41-
headers['x-account'] = JSON.stringify(account)
42-
}
43-
headers['x-processing'] = JSON.stringify({ _id: processing._id, title: encodeURIComponent(processing.title) })
44-
45-
const axiosInstance = axios.create({
46-
// this is necessary to prevent excessive memory usage during large file uploads, see https://github.com/axios/axios/issues/1045
47-
maxRedirects: 0,
48-
httpAgent,
49-
httpsAgent
50-
})
51-
52-
// apply default base url and send api key when relevant
53-
axiosInstance.interceptors.request.use(cfg => {
54-
if (!cfg.url) throw new Error('missing url in axios request')
55-
if (!/^https?:\/\//i.test(cfg.url)) {
56-
if (cfg.url.startsWith('/')) cfg.url = config.dataFairUrl + cfg.url
57-
else cfg.url = config.dataFairUrl + '/' + cfg.url
58-
}
59-
const isDataFairUrl = cfg.url.startsWith(config.dataFairUrl)
60-
if (isDataFairUrl) Object.assign(cfg.headers, headers)
61-
cfg.headers['User-Agent'] = cfg.headers['User-Agent'] ?? `@data-fair/processings (${processing.plugin})`
62-
63-
// use private data fair url if specified to prevent leaving internal infrastructure
64-
// except from GET requests so that they still appear in metrics
65-
// except if config.getFromPrivateDataFairUrl is set to true, then all requests are sent to the private url
66-
const usePrivate =
67-
config.privateDataFairUrl &&
68-
isDataFairUrl &&
69-
(config.getFromPrivateDataFairUrl || ['post', 'put', 'delete', 'patch'].includes(cfg.method || ''))
70-
if (usePrivate) {
71-
cfg.url = cfg.url.replace(config.dataFairUrl, config.privateDataFairUrl!)
72-
cfg.headers.host = new URL(config.dataFairUrl).host
73-
}
74-
return cfg
75-
}, error => Promise.reject(error))
76-
77-
return axiosInstance
78-
}
79-
80-
// customize axios errors for shorter stack traces when a request fails
81-
// WARNING: we used to do it in an interceptor, but it was incompatible with axios-retry
82-
const prepareAxiosError = (error: any) => {
83-
const response = error.response ?? error.request?.res ?? error.res
84-
if (!response) return error
85-
delete response.request
86-
const headers: Record<string, string> = {}
87-
if (response.headers?.location) headers.location = response.headers.location
88-
response.headers = headers
89-
response.config = response.config ?? error.config
90-
if (response.config) {
91-
response.config = { method: response.config.method, url: response.config.url, params: response.config.params, data: response.config.data }
92-
if (response.config.data && response.config.data._writableState) delete response.config.data
93-
}
94-
if (response.data && response.data._readableState) delete response.data
95-
if (error.message) response.message = error.message
96-
if (error.stack) response.stack = error.stack
97-
return response
98-
}
99-
10028
/**
10129
* Create a WebSocket instance.
10230
*/
@@ -171,17 +99,7 @@ export const run = async (mailTransport: any) => {
17199
const tmpDir = await tmp.dir({ unsafeCleanup: true, tmpdir: baseTmpDir, prefix: `processing-run-${processing._id}-${run._id}` })
172100
const processingConfig = processing.config || {}
173101

174-
const axiosInstance = getAxiosInstance(processing)
175-
axiosRetry(axiosInstance, {
176-
retries: 3,
177-
retryDelay: axiosRetry.exponentialDelay,
178-
shouldResetTimeout: true,
179-
onRetry: (retryCount, _err, requestConfig) => {
180-
const err = prepareAxiosError(_err)
181-
const message = getHttpErrorMessage(err) || err.message || err
182-
log.warning(`tentative ${retryCount} de requête ${requestConfig.method} ${requestConfig.url} : ${message}`)
183-
}
184-
})
102+
const axiosInstance = getAxiosInstance(processing, log)
185103

186104
const secrets: Record<string, string> = {}
187105
if (processing.secrets) {
@@ -254,27 +172,4 @@ export const stop = async () => {
254172
await new Promise(resolve => setTimeout(resolve, config.worker.gracePeriod))
255173
}
256174

257-
const getHttpErrorMessage = (err: any) => {
258-
let httpMessage = err.status ?? err.statusCode
259-
if (httpMessage) {
260-
const statusText = err.statusText ?? err.statusMessage
261-
if (statusText) httpMessage += ' - ' + statusText
262-
if (err.data) {
263-
if (typeof err.data === 'string') httpMessage += ' - ' + err.data
264-
else httpMessage += ' - ' + JSON.stringify(err.data)
265-
} else if (err.message) {
266-
httpMessage += ' - ' + err.message
267-
}
268-
if (err.config && err.config.url) {
269-
let url = err.config.url
270-
url = url.replace(config.dataFairUrl, '')
271-
if (config.privateDataFairUrl) {
272-
url = url.replace(config.privateDataFairUrl, '')
273-
}
274-
httpMessage += ` (${url})`
275-
}
276-
return httpMessage
277-
}
278-
}
279-
280175
export default { run, stop }

0 commit comments

Comments
 (0)