Skip to content

Commit 27cc974

Browse files
DavideTarditiDavideTarditi
committed
feat: init upload form ui, improve csv parsing algorythm, and refactor
1 parent b6e7694 commit 27cc974

File tree

19 files changed

+677
-112
lines changed

19 files changed

+677
-112
lines changed

‎apps/api/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"ts-node": "^10.9.2",
3737
"typescript": "^5.7.3",
3838
"ua-parser-js": "^2.0.1",
39+
"uuid": "^9.0.1",
3940
"winston": "^3.17.0"
4041
},
4142
"devDependencies": {

‎apps/api/src/api/v1/controllers/transaction.controller.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ import { Request, Response } from 'express'
22
import prisma from '@poveroh/prisma'
33
import { TransactionHelper } from '../helpers/transaction.helper'
44
import { buildWhere } from '../../../helpers/filter.helper'
5-
import { IFilterOptions, ITransactionFilters } from '@poveroh/types'
5+
import { ICsvReadedTransaction, IFilterOptions, ITransactionFilters, TransactionAction } from '@poveroh/types'
66
import logger from '../../../utils/logger'
7+
import CSVParser from '../helpers/csv.helper'
78

89
export class TransactionController {
910
//POST /
@@ -117,4 +118,30 @@ export class TransactionController {
117118
res.status(500).json({ message: 'An error occurred', error })
118119
}
119120
}
121+
122+
//POST /read-csv
123+
static async parseCSV(req: Request, res: Response) {
124+
try {
125+
if (!req.files || req.files.length === 0) throw new Error('Data not provided')
126+
127+
const files = req.files as Express.Multer.File[]
128+
const parser = new CSVParser()
129+
130+
const results: ICsvReadedTransaction[] = []
131+
for (const file of files) {
132+
const content = file.buffer.toString('utf-8')
133+
134+
const res = await parser.parseCSVFile(content)
135+
136+
results.push(...res.transactions)
137+
}
138+
139+
const parsedTransactions = TransactionHelper.normalizeTransaction(req.user.id, '', results)
140+
141+
res.status(200).json(parsedTransactions)
142+
} catch (error) {
143+
logger.error(error)
144+
res.status(500).json({ message: 'An error occurred', error })
145+
}
146+
}
120147
}

‎apps/api/src/api/v1/helpers/csv.helper.ts‎

Lines changed: 239 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import Papa from 'papaparse'
2-
import _ from 'lodash'
3-
import { ICsvReadedTransaction, IFieldMapping, TransactionAction, Currencies } from '@poveroh/types'
2+
import { ICsvReadedTransaction, IFieldMapping, TransactionAction, Currencies, ICSVValueReturned } from '@poveroh/types'
43

54
class CSVParser {
65
private datePatterns = [
@@ -53,7 +52,92 @@ class CSVParser {
5352
'transaction',
5453
'details',
5554
'product',
56-
'legenda'
55+
'legenda',
56+
'remark',
57+
'text',
58+
'transaction type',
59+
'item',
60+
'purpose',
61+
'label',
62+
'activity',
63+
'particulars',
64+
'explanation',
65+
'subject',
66+
'category',
67+
'origin',
68+
'destination',
69+
'info',
70+
'transaktionstext',
71+
'betreff',
72+
'Verwendungszweck',
73+
'libelle',
74+
'intitulé',
75+
'descripción',
76+
'concepto',
77+
'referencia',
78+
'historia',
79+
'articolo',
80+
'causale',
81+
'ragione sociale',
82+
'beneficiary',
83+
'payer',
84+
'counterparty',
85+
'transaction id',
86+
'trans id',
87+
'operation',
88+
'reason',
89+
'comment',
90+
'type',
91+
'channel',
92+
'document',
93+
'invoice',
94+
'receipt',
95+
'statement',
96+
'intestatario',
97+
'beneficiario',
98+
'ordinante',
99+
'movimento',
100+
'operazione',
101+
'causale ABI',
102+
'causale SIA',
103+
'tipo operazione',
104+
'descrizione operazione',
105+
'dettaglio',
106+
'trans details',
107+
'trans description',
108+
'account name',
109+
'bank name',
110+
'branch',
111+
'location',
112+
'description 1',
113+
'description 2',
114+
'narrative',
115+
'posting text',
116+
'source',
117+
'recipient',
118+
'sender',
119+
'payee name',
120+
'description of goods',
121+
'order id',
122+
'billing ref',
123+
'clearing text',
124+
'conto',
125+
'codice operazione',
126+
'riferimento',
127+
'concept',
128+
'designation',
129+
'item description',
130+
'line item',
131+
'transaction detail',
132+
'payment details',
133+
'transaction reference',
134+
'payment reference',
135+
'transaction info',
136+
'trans info',
137+
'bank reference',
138+
'customer reference',
139+
'segno',
140+
'descrizione estesa'
57141
]
58142

59143
private dateKeywords = [
@@ -66,7 +150,58 @@ class CSVParser {
66150
'started',
67151
'when',
68152
'day',
69-
'datetime'
153+
'datetime',
154+
'posting date',
155+
'transaction date',
156+
'effective date',
157+
'value date',
158+
'settlement date',
159+
'processing date',
160+
'activity date',
161+
'booked date',
162+
'entry date',
163+
'execution date',
164+
'journal date',
165+
'ora',
166+
'giorno',
167+
'fecha',
168+
'hora',
169+
'datum',
170+
'uhrzeit',
171+
'date operation',
172+
'date valeur',
173+
'date comptable',
174+
'jour',
175+
'transactiontime',
176+
'system date',
177+
'record date',
178+
'payment date',
179+
'emission date',
180+
'scadenza',
181+
'data contabile',
182+
'data valuta',
183+
'data operazione',
184+
'data esecuzione',
185+
'transfer date',
186+
'report date',
187+
'received date',
188+
'sent date',
189+
'cut-off date',
190+
'statement date',
191+
'period start',
192+
'period end',
193+
'data di accredito',
194+
'data di addebito',
195+
'data registrazione',
196+
'start date',
197+
'end date',
198+
'due date',
199+
'occurrence date',
200+
'transaction_date',
201+
'posting_date',
202+
'valuta',
203+
'data movimento',
204+
'data contabilizzazione'
70205
]
71206

72207
private amountKeywords = [
@@ -81,7 +216,89 @@ class CSVParser {
81216
'charge',
82217
'balance',
83218
'debit',
84-
'credit'
219+
'credit',
220+
'deposit',
221+
'withdrawal',
222+
'outflow',
223+
'inflow',
224+
'paid',
225+
'received',
226+
'currency',
227+
'valore',
228+
'prezzo',
229+
'totale',
230+
'saldo',
231+
'addebito',
232+
'accredito',
233+
'montante',
234+
'somme',
235+
'débit',
236+
'crédit',
237+
'solde',
238+
'importe',
239+
'saldo',
240+
'débito',
241+
'crédito',
242+
'betrag',
243+
'summe',
244+
'solde',
245+
'soll',
246+
'haben',
247+
'net amount',
248+
'gross amount',
249+
'principal',
250+
'interest',
251+
'tax',
252+
'payment',
253+
'return',
254+
'refund',
255+
'disbursement',
256+
'remittance',
257+
'exchange',
258+
'commission',
259+
'discount',
260+
'vat',
261+
'tva',
262+
'imposta',
263+
'spese',
264+
'interessi',
265+
'capitale',
266+
'salario',
267+
'stipendio',
268+
'rata',
269+
'ammontare',
270+
'due amount',
271+
'final amount',
272+
'subtotal',
273+
'grand total',
274+
'transaction amount',
275+
'transfer amount',
276+
'loan amount',
277+
'initial amount',
278+
'remaining balance',
279+
'current balance',
280+
'previous balance',
281+
'commissione',
282+
'prelievo',
283+
'versamento',
284+
'rata capitale',
285+
'rata interessi',
286+
'oneri',
287+
'lordo',
288+
'totale lordo',
289+
'totale netto',
290+
'importo originale',
291+
'amount_usd',
292+
'amount_eur',
293+
'total_amount',
294+
'current_balance',
295+
'available balance',
296+
'book balance',
297+
'posted amount',
298+
'transaction_amount',
299+
'entry amount',
300+
'valore nominale',
301+
'differenza'
85302
]
86303

87304
private detectFields(headers: string[], sampleRows: Record<string, any>[]): IFieldMapping {
@@ -218,8 +435,8 @@ class CSVParser {
218435
return isNaN(parsed) ? 0 : parsed
219436
}
220437

221-
private parseDate(value: string): string {
222-
if (!value) return new Date().toISOString().split('T')[0]
438+
private parseDate(value: string): Date {
439+
if (!value) return new Date()
223440

224441
const attempts = [
225442
() => new Date(value),
@@ -267,35 +484,40 @@ class CSVParser {
267484
try {
268485
const date = attempt()
269486
if (date instanceof Date && !isNaN(date.getTime())) {
270-
return date.toISOString().split('T')[0]
487+
return date
271488
}
272489
} catch {
273490
continue
274491
}
275492
}
276493

277494
console.warn(`Unable to parse date: ${value}, using current date`)
278-
return new Date().toISOString().split('T')[0]
495+
return new Date()
279496
}
280497

281-
private extractCurrency(row: Record<string, any>, currencyField?: string): string {
498+
private extractCurrency(row: Record<string, any>, currencyField?: string): Currencies {
282499
if (currencyField && row[currencyField]) {
283-
const currency = String(row[currencyField]).trim()
500+
const currency = String(row[currencyField])
501+
.trim()
502+
.replace(/[^A-Z€$£¥₹₽]/g, '')
284503
if (this.isLikelyCurrency(currency)) {
285-
return currency.replace(/[^A-Z€$£¥₹₽]/g, '')
504+
const match = Currencies[currency as keyof typeof Currencies]
505+
return match ?? Currencies.UNKNOWN
286506
}
287507
}
288508

289-
for (const [key, value] of Object.entries(row)) {
509+
for (const [_, value] of Object.entries(row)) {
290510
if (value && this.isLikelyCurrency(String(value))) {
291511
const match = String(value).match(/EUR|USD|GBP|JPY|CHF|CAD|AUD|€|\$|£|¥|₹|₽/i)
292512
if (match) {
293-
return match[0].toUpperCase()
513+
const cleaned = match[0].toUpperCase().replace(/[^A-Z€$£¥₹₽]/g, '')
514+
const matchCurrency = Currencies[cleaned as keyof typeof Currencies]
515+
return matchCurrency ?? Currencies.UNKNOWN
294516
}
295517
}
296518
}
297519

298-
return 'UNKNOWN'
520+
return Currencies.UNKNOWN
299521
}
300522

301523
private findDataTableStart(csvData: string): {
@@ -495,19 +717,7 @@ class CSVParser {
495717
})
496718
}
497719

498-
public async parseCSVFile(fileContent: string): Promise<{
499-
transactions: ICsvReadedTransaction[]
500-
mapping: IFieldMapping
501-
errors: string[]
502-
detectedStartRow?: number
503-
summary: {
504-
totalTransactions: number
505-
totalIncome: number
506-
totalExpenses: number
507-
dateRange: { from: string; to: string }
508-
currencies: string[]
509-
}
510-
}> {
720+
public async parseCSVFile(fileContent: string): Promise<ICSVValueReturned> {
511721
const result = await this.parseCSV(fileContent)
512722

513723
const summary = {
@@ -517,12 +727,7 @@ class CSVParser {
517727
.reduce((sum, t) => sum + t.amount, 0),
518728
totalExpenses: result.transactions
519729
.filter(t => t.type === TransactionAction.EXPENSES)
520-
.reduce((sum, t) => sum + t.amount, 0),
521-
dateRange: {
522-
from: result.transactions.length > 0 ? _.min(result.transactions.map(t => t.date)) || '' : '',
523-
to: result.transactions.length > 0 ? _.max(result.transactions.map(t => t.date)) || '' : ''
524-
},
525-
currencies: _.uniq(result.transactions.map(t => t.currency))
730+
.reduce((sum, t) => sum + t.amount, 0)
526731
}
527732

528733
return { ...result, summary }

0 commit comments

Comments
 (0)