diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0f8a7703..2721fb7c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,14 +16,14 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '>=1.20.0' + go-version: '>=1.26' check-latest: true - name: Build run: go build -v ./... - name: Test - run: go test -v -coverprofile=profile.cov . ./decimal + run: go test -v -coverprofile=profile.cov . - uses: shogo82148/actions-goveralls@v1 with: diff --git a/.gitignore b/.gitignore index 18ef2420..ff8971eb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ pkg dist html-book src-book +.* diff --git a/balances.go b/balances.go index 6f539a3b..6a06e0a4 100644 --- a/balances.go +++ b/balances.go @@ -4,7 +4,7 @@ import ( "slices" "strings" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) // GetBalances provided a list of transactions and filter strings, returns account balances of @@ -14,13 +14,13 @@ import ( // Accounts are sorted by name. func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { var accList []*Account - balances := make(map[string]*Account) + balances := make(map[string]map[string]*Account) // at every depth, for each account, track the parent account depthMap := make(map[int]map[string]string) var maxDepth int - incAccount := func(accName string, val decimal.Decimal) { + incAccount := func(accName string, currency string, val decimal.Decimal) { // track parent accDepth := strings.Count(accName, ":") + 1 pmap, pmapfound := depthMap[accDepth] @@ -35,10 +35,14 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { } // add to balance - if acc, ok := balances[accName]; !ok { - acc := &Account{Name: accName, Balance: val} + if _, ok := balances[accName]; !ok { + balances[accName] = make(map[string]*Account) + } + + if acc, ok := balances[accName][currency]; !ok { + acc := &Account{Name: accName, Currency: currency, Balance: val} accList = append(accList, acc) - balances[accName] = acc + balances[accName][currency] = acc } else { acc.Balance = acc.Balance.Add(val) } @@ -53,7 +57,7 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { } } if inFilter { - incAccount(accChange.Name, accChange.Balance) + incAccount(accChange.Name, accChange.Currency, accChange.Balance) } } } @@ -61,7 +65,9 @@ func GetBalances(generalLedger []*Transaction, filterArr []string) []*Account { // roll-up balances for curDepth := maxDepth; curDepth > 1; curDepth-- { for accName, parentName := range depthMap[curDepth] { - incAccount(parentName, balances[accName].Balance) + for currency, acc := range balances[accName] { + incAccount(parentName, currency, acc.Balance) + } } } diff --git a/balances_test.go b/balances_test.go index 927f8918..14fe68c7 100644 --- a/balances_test.go +++ b/balances_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) type testBalCase struct { @@ -71,12 +71,58 @@ var testBalCases = []testBalCase{ }, nil, }, + { + "conversion", + `2026/01/21 Converted CZK to EUR + CZK -2000.00 @ 0.5 + EUR 1000.00 + +2026/01/21 Converted CZK to EUR + CZK -2000.00 @@ 1000.00 + EUR 1000.00 +`, + []Account{ + { + Name: "CZK", + Balance: decimal.NewFromFloat(-4000), + }, + { + Name: "EUR", + Balance: decimal.NewFromFloat(2000), + }, + }, + nil, + }, + { + "conversion", + `2026/01/21 Converted CZK to EUR + CZK CZK -2000.00 @ 0.5 + EUR EUR 1000.00 + +2026/01/21 Converted CZK to EUR + CZK CZK -2000.00 @@ 1000.00 + EUR EUR 1000.00 +`, + []Account{ + { + Name: "CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-4000), + }, + { + Name: "EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(2000), + }, + }, + nil, + }, } func TestBalanceLedger(t *testing.T) { for _, tc := range testBalCases { b := bytes.NewBufferString(tc.data) - transactions, err := ParseLedger(b) + transactions, err := ParseLedger("", b) bals := GetBalances(transactions, []string{}) if (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { t.Errorf("Error: expected `%s`, got `%s`", tc.err, err) @@ -143,7 +189,7 @@ func TestBalancesByPeriod(t *testing.T) { `) - trans, _ := ParseLedger(b) + trans, _ := ParseLedger("", b) partitionRb := BalancesByPeriod(trans, PeriodQuarter, RangePartition) snapshotRb := BalancesByPeriod(trans, PeriodQuarter, RangeSnapshot) diff --git a/decimal/decimal.go b/decimal/decimal.go deleted file mode 100644 index 16f7fcbc..00000000 --- a/decimal/decimal.go +++ /dev/null @@ -1,314 +0,0 @@ -// Package decimal implements fixed-point decimal with accuracy to 3 digits of -// precision after the decimal point. -// -// int64 is the underlying data type for speed of computation. However, using -// an int64 cast to Decimal will not work, one of the "New" functions must -// be used to get accurate results. -// -// The package multiplies every source value by 1000, and then does integer -// math from that point forward, maintaining all values at that scale over -// every operation. -// -// Note: For use in ledger. Cannot handle values over approx 900 trillion. -package decimal - -import ( - "errors" - "strconv" - "strings" -) - -// Decimal represents a fixed-point decimal. -type Decimal int64 - -// scaleFactor used for math operations, -const scaleFactor = 1000 - -// precision of 3 digits -const precision = 3 - -// Zero constant, to make initializations easier. -const Zero = Decimal(0) - -// One constant, to make initializations easier. -const One = Decimal(scaleFactor) - -// Parse max/min for whole number part -const parseMax = (1<<63 - 1) / scaleFactor -const parseMin = (-1 << 63) / scaleFactor - -// NewFromFloat converts a float64 to Decimal. Only 3 digits of precision after -// the decimal point are preserved. -func NewFromFloat(f float64) Decimal { - return Decimal(f * float64(scaleFactor)) -} - -// NewFromInt converts a int64 to Decimal. Multiplies by 1000 to get into -// Decimal scale. -func NewFromInt(i int64) Decimal { - return Decimal(i) * scaleFactor -} - -var errEmpty = errors.New("empty string") -var errTooBig = errors.New("number too big") -var errInvalid = errors.New("invalid syntax") - -// atoi64 is equivalent to strconv.Atoi -func atoi64(s string) (bool, int64, error) { - sLen := len(s) - if sLen < 1 { - return false, 0, errEmpty - } - if sLen > 18 { - return false, 0, errTooBig - } - - neg := false - if s[0] == '-' { - neg = true - s = s[1:] - if len(s) < 1 { - return neg, 0, errEmpty - } - } - - var n int64 - for _, ch := range []byte(s) { - ch -= '0' - if ch > 9 { - return neg, 0, errInvalid - } - n = n*10 + int64(ch) - } - if neg { - n = -n - } - return neg, n, nil -} - -// NewFromString returns a Decimal from a string representation. Throws an -// error if integer parsing fails. -func NewFromString(s string) (Decimal, error) { - if whole, frac, split := strings.Cut(s, "."); split { - neg, w, err := atoi64(whole) - // if fractional portion exists, whole part can be empty - if err != nil && err != errEmpty { - return Zero, err - } - - // overflow - if w > parseMax || w < parseMin { - return Zero, errTooBig - } - w *= int64(scaleFactor) - - // Parse up to *precision* digits and scale up - var f int64 - var seen int - for _, b := range frac { - f *= 10 - if b < '0' || b > '9' { - return Zero, errInvalid - } - f += int64(b - '0') - seen++ - if seen == precision { - break - } - } - for seen < precision { - f *= 10 - seen++ - } - - if neg { - f = -f - } - return Decimal(w + f), nil - } - - _, i, err := atoi64(s) - if i > parseMax || i < parseMin { - return Zero, errTooBig - } - i *= int64(scaleFactor) - return Decimal(i), err -} - -// IsZero returns true if d == 0 -func (d Decimal) IsZero() bool { - return d == Zero -} - -// Neg returns -d -func (d Decimal) Neg() Decimal { - return -d -} - -// Sign returns: -// -// -1 if d < 0 -// -// 0 if d == 0 -// -// +1 if d > 0 -func (d Decimal) Sign() int { - if d < 0 { - return -1 - } else if d > 0 { - return 1 - } - return 0 -} - -// Add returns d + d1 -func (d Decimal) Add(d1 Decimal) Decimal { - return d + d1 -} - -// Sub returns d - d1 -func (d Decimal) Sub(d1 Decimal) Decimal { - return d - d1 -} - -// Mul returns d * d1 -func (d Decimal) Mul(d1 Decimal) Decimal { - return (d * d1) / scaleFactor -} - -// Div returns d / d1 -func (d Decimal) Div(d1 Decimal) Decimal { - return (d * scaleFactor) / d1 -} - -// Abs returns the absolute value of the decimal -func (d Decimal) Abs() Decimal { - if d < 0 { - return d.Neg() - } - return d -} - -// Float64 returns the float64 value for d, and exact is always set to false. -// The signature is this way to match big.Rat -func (d Decimal) Float64() (f float64, exact bool) { - return float64(d) / float64(scaleFactor), false -} - -// Cmp compares the numbers represented by d and d1 and returns: -// -// -1 if d < d1 -// 0 if d == d1 -// +1 if d > d1 -func (d Decimal) Cmp(d1 Decimal) int { - if d < d1 { - return -1 - } else if d > d1 { - return 1 - } - return 0 -} - -// fmtFrac formats the fraction of v/10**prec (e.g., ".12345") into the -// tail of buf. It returns the index where the -// output bytes begin and the value v/10**prec. -func fmtFrac(buf []byte, v uint64, prec int) (nw int, nv uint64) { - w := len(buf) - for range prec { - digit := v % 10 - w-- - buf[w] = byte(digit) + '0' - v /= 10 - } - w-- - buf[w] = '.' - return w, v -} - -// fmtInt formats v into the tail of buf. -// It returns the index where the output begins. -func fmtInt(buf []byte, v uint64) int { - w := len(buf) - if v == 0 { - w-- - buf[w] = '0' - } else { - for v > 0 { - w-- - buf[w] = byte(v%10) + '0' - v /= 10 - } - } - return w -} - -// StringFixedBank returns a banker rounded fixed-point string with 2 digits -// after the decimal point. -// -// Example: -// -// NewFromFloat(5.455).StringFixedBank() == "5.46" -// NewFromFloat(5.445).StringFixedBank() == "5.44" -func (d Decimal) StringFixedBank() string { - var buf [24]byte - w := len(buf) - - u := uint64(d) - neg := d < 0 - if neg { - u = -u - } - - // Bank rounding - rem := u % 10 - u /= 10 - if rem > 5 || (rem == 5 && u%2 != 0) { - u++ - } - - // fmt functions from time.Duration - w, u = fmtFrac(buf[:w], u, precision-1) - w = fmtInt(buf[:w], u) - - if neg { - w-- - buf[w] = '-' - } - - return string(buf[w:]) -} - -// StringTruncate returns the whole-number (Int) part of d. -// -// Example: -// -// NewFromFloat(5.44).StringTruncate() == "5" -func (d Decimal) StringTruncate() string { - whole := d / scaleFactor - return strconv.FormatInt(int64(whole), 10) -} - -// StringRound returns the nearest rounded whole-number (Int) part of d. -// Example: -// -// NewFromFloat(5.5).StringRound() == "6" -// NewFromFloat(5.4).StringRound() == "5" -// NewFromFloat(-5.4).StringRound() == "5" -// NewFromFloat(-5.5).StringRound() == "6" -func (d Decimal) StringRound() string { - whole := d / scaleFactor - frac := (d % scaleFactor) - neg := false - if frac < 0 { - frac = -frac - neg = true - } - if frac >= (5 * (scaleFactor / 10)) { - if neg { - whole-- - } else { - whole++ - } - } - return strconv.FormatInt(int64(whole), 10) -} diff --git a/decimal/decimal_test.go b/decimal/decimal_test.go deleted file mode 100644 index 327125c8..00000000 --- a/decimal/decimal_test.go +++ /dev/null @@ -1,437 +0,0 @@ -package decimal - -import ( - "math/rand" - "strings" - "testing" - - sdec "github.com/shopspring/decimal" -) - -type testCase struct { - name string - Result, Input string -} - -var testCases = []testCase{ - { - "multiply", - NewFromFloat(48.0).StringFixedBank(), - NewFromInt(6).Mul(NewFromInt(8)).StringFixedBank(), - }, - { - "divide", - NewFromFloat(6.0).StringFixedBank(), - NewFromInt(48).Div(NewFromInt(8)).StringFixedBank(), - }, - { - "divide-1", - NewFromFloat(11.111).StringFixedBank(), - NewFromInt(100).Div(NewFromInt(9)).StringFixedBank(), - }, - { - "sum", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(123.12).Add(NewFromInt(111)).Add(NewFromFloat(0.44)).StringFixedBank(), - }, - { - "bankrounduppos", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(234.555).StringFixedBank(), - }, - { - "bankrounddownpos", - NewFromFloat(234.54).StringFixedBank(), - NewFromFloat(234.545).StringFixedBank(), - }, - { - "bankroundupneg", - "-234.56", - NewFromFloat(-234.555).StringFixedBank(), - }, - { - "bankrounddownneg", - "-234.54", - NewFromFloat(-234.545).StringFixedBank(), - }, - { - "rounduppos", - NewFromFloat(234.56).StringFixedBank(), - NewFromFloat(234.556).StringFixedBank(), - }, - { - "rounddownpos", - NewFromFloat(234.55).StringFixedBank(), - NewFromFloat(234.554).StringFixedBank(), - }, - { - "roundupneg", - "-234.56", - NewFromFloat(-234.556).StringFixedBank(), - }, - { - "rounddownneg", - "-234.55", - NewFromFloat(-234.554).StringFixedBank(), - }, - { - "truncate", - NewFromInt(234).StringTruncate(), - NewFromFloat(234.554).StringTruncate(), - }, - { - "2digits-1", - "1.00", - One.StringFixedBank(), - }, - { - "2digits-4.5", - "4.50", - NewFromFloat(4.5).StringFixedBank(), - }, - { - "roundintuppos", - "6", - NewFromFloat(5.6).StringRound(), - }, - { - "roundintdownpos", - "5", - NewFromFloat(5.4).StringRound(), - }, - { - "roundintupneg", - "-5", - NewFromFloat(-5.4).StringRound(), - }, - { - "roundintdownneg", - "-6", - NewFromFloat(-5.6).StringRound(), - }, - { - "negfrac", - "-0.43", - NewFromFloat(-0.43).StringFixedBank(), - }, - { - "sub", - "5.12", - NewFromFloat(5.56).Sub(NewFromFloat(0.44)).StringFixedBank(), - }, - { - "neg", - "-5.12", - NewFromFloat(5.12).Neg().StringFixedBank(), - }, - { - "abs-1", - "5.12", - NewFromFloat(-5.12).Abs().StringFixedBank(), - }, - { - "abs-1", - "5.12", - NewFromFloat(5.12).Abs().StringFixedBank(), - }, -} - -func TestDecimal(t *testing.T) { - for _, tc := range testCases { - if tc.Result != tc.Input { - t.Errorf("Error(%s): expected \n`%s`, \ngot \n`%s`", tc.name, tc.Result, tc.Input) - } - } -} - -func TestFloat(t *testing.T) { - d := NewFromFloat(5.56) - f := float64(5.56) - if df, _ := d.Float64(); df != f { - t.Error("Float64 not exact") - } -} - -func TestCompare(t *testing.T) { - l := NewFromInt(5) - h := NewFromInt(10) - z := NewFromInt(0) - - if !z.IsZero() { - t.Error("zero failed") - } - - if h.Cmp(l) != 1 || l.Cmp(h) != -1 || z.Cmp(Zero) != 0 { - t.Error("compare fail") - } -} - -func TestSign(t *testing.T) { - n := NewFromInt(-5) - p := NewFromInt(5) - z := NewFromInt(0) - - if z.Sign() != 0 { - t.Error("zero failed") - } - - if n.Sign() != -1 || p.Sign() != 1 { - t.Error("sign fail") - } -} - -var testParseCases = []testCase{ - { - "negzero", - "-0.43", - "-0.43", - }, - { - "poszero", - "0.43", - "0.43", - }, - { - "3digit", - "5.56", - "5.564", - }, - { - "truncateinput", - "5.56", - "5.56432342", - }, - { - "precise", - "16.24", - "16.24", - }, - { - "fuzz-1", - "0.00", - "0.0051", - }, - { - "fuzz-2", - "8.00", - "8.005", - }, - { - "fuzz-3", - "0.00", - "0.005", - }, - { - "fuzz-4", - "1.00", - "0.997", - }, - { - "fuzz-5", - "2200000000000021.00", - "2200000000000021", - }, - { - "fuzz-6", - "0.01", - "0.010e1", - }, - { - "fuzz-7", - "-8.00", - "-7.995", - }, - { - "fuzz-8", - "-9.00", - "-8.995", - }, - { - "fuzz-9", - "8.00", - "7.995", - }, - { - "fuzz-10", - "9.00", - "8.995", - }, - { - "fuzz-11", - "-7.98", - "-7.985", - }, - { - "fuzz-12", - "-8.98", - "-8.985", - }, - { - "fuzz-13", - "7.98", - "7.985", - }, - { - "fuzz-14", - "8.98", - "8.984", - }, - { - "fuzz-15", - "-8.00", - "-7.999", - }, - { - "fuzz-16", - "-9.00", - "-8.999", - }, - { - "fuzz-17", - "8.00", - "7.999", - }, - { - "fuzz-18", - "9.00", - "8.999", - }, - { - "error-1", - errTooBig.Error(), - "100000000000000000", - }, - { - "error-2", - errTooBig.Error(), - "10000000000000000", - }, - { - "error-3", - errTooBig.Error(), - "10000000000000000.56", - }, - { - "error-4", - errInvalid.Error(), - "0.e0", - }, - { - "error-5", - errTooBig.Error(), - "5555555555555555555555555550000000000000000", - }, - { - "error-6", - errEmpty.Error(), - "-", - }, - { - "error-7", - errEmpty.Error(), - "", - }, - { - "error-badint-1", - errInvalid.Error(), - "1QZ.56", - }, - { - "error-expr-1", - errInvalid.Error(), - "(123 * 6)", - }, - { - "missingwhole", - "0.50", - ".50", - }, - { - "negmissingwhole", - "-0.50", - "-.50", - }, - { - "missingfrac", - "5.00", - "5.", - }, - { - "neg-missingfrac", - "-5.00", - "-5.", - }, - { - "just-a-decimal", - "0.00", - ".", - }, -} - -func TestStringParse(t *testing.T) { - for _, tc := range testParseCases { - d, err := NewFromString(tc.Input) - if strings.HasPrefix(tc.name, "error") { - if err == nil { - t.Fatalf("Error(%s): expected error `%s`", tc.name, tc.Result) - } - if err.Error() != tc.Result { - t.Fatalf("Error(%s): expected `%s`, got `%s`", tc.name, tc.Result, err) - } - } - if !strings.HasPrefix(tc.name, "error") && err != nil { - t.Fatalf("Error(%s): unexpected error `%s`", tc.name, err) - } - if !strings.HasPrefix(tc.name, "error") && tc.Result != d.StringFixedBank() { - t.Errorf("Error(%s): expected \n`%s`, \ngot \n`%s`", tc.name, tc.Result, d.StringFixedBank()) - } - } -} - -func FuzzStringParse(f *testing.F) { - f.Fuzz(func(t *testing.T, s string) { - if _, after, split := strings.Cut(s, "."); split { - if len(after) > 3 { - return - } - } - sd, serr := sdec.NewFromString(s) - if serr != nil { - return - } - d, err := NewFromString(s) - if err != nil { - return - } - ss := strings.TrimPrefix(sd.StringFixedBank(2), "-") - ds := strings.TrimPrefix(d.StringFixedBank(), "-") - - if ds != ss { - t.Fatalf("no match: decimal \n`%s`, \nsdec \n `%s`", ds, ss) - } - }) -} - -func BenchmarkNewFromString(b *testing.B) { - numbers := []string{"10.0", "245.6", "354", "2.456", "-31.2"} - for b.Loop() { - for _, numStr := range numbers { - NewFromString(numStr) - } - } -} - -func BenchmarkStringFixedBank(b *testing.B) { - var numbers [1000]Decimal - for i := range len(numbers) { - numbers[i] = NewFromFloat(rand.Float64() * 100000) - if i%2 == 0 { - numbers[i] *= -1 - } - } - for b.Loop() { - for _, num := range numbers { - num.StringFixedBank() - } - } -} diff --git a/go.mod b/go.mod index 81e1912e..2142c904 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,23 @@ module github.com/howeyc/ledger -go 1.22 +go 1.26 + +toolchain go1.26.0 require ( - github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976 github.com/andybalholm/brotli v1.0.6 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de + github.com/expr-lang/expr v1.17.8 github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b github.com/ivanpirog/coloredcobra v1.0.1 github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a - github.com/joyt/godate v0.0.0-20150226210126-7151572574a7 github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2 github.com/lucasb-eyer/go-colorful v1.2.0 github.com/mattn/go-isatty v0.0.20 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml v1.9.5 - github.com/shopspring/decimal v1.3.1 + github.com/samber/lo v1.53.0 + github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.7.0 golang.org/x/term v0.13.0 golang.org/x/time v0.3.0 @@ -26,4 +29,5 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 3527d845..b1f66b35 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ -github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976 h1:+jyVKPjl5Y39thM0ZlVrRqKjSO/Upr5tP9ZQGELv8gw= -github.com/alfredxing/calc v0.0.0-20180827002445-77daf576f976/go.mod h1:/HQknSiD7YKT15DoHXuiXezQfNPBUm8PeqFaTxeA3HU= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/expr-lang/expr v1.17.8 h1:W1loDTT+0PQf5YteHSTpju2qfUfNoBt4yw9+wOEU9VM= +github.com/expr-lang/expr v1.17.8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= @@ -16,8 +20,6 @@ github.com/ivanpirog/coloredcobra v1.0.1 h1:aURSdEmlR90/tSiWS0dMjdwOvCVUeYLfltLf github.com/ivanpirog/coloredcobra v1.0.1/go.mod h1:iho4nEKcnwZFiniGSdcgdvRgZNjxm+h20acv8vqmN6Q= github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a h1:gbdjhSslIoRRiSSLCP3kKuLmqAJGmhnPVhIyf6Dbw34= github.com/jbrukh/bayesian v0.0.0-20200318221351-d726b684ca4a/go.mod h1:SELxwZQq/mPnfPCR2mchLmT4TQaPJvYtLcCtDWSM7vM= -github.com/joyt/godate v0.0.0-20150226210126-7151572574a7 h1:2wH5antjhmU3EuWyidm0lJ4B9hGMpl5lNRo+M9uGJ5A= -github.com/joyt/godate v0.0.0-20150226210126-7151572574a7/go.mod h1:R+UgFL3iylLhx9N4w35zZ2HdhDlgorRDx4SxbchWuN0= github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2 h1:jrs0oyU9XY7MlTHbNxecqFgY+fgEENZdP4Z8FZln/pw= github.com/juztin/numeronym v0.0.0-20160223091026-859fcc2918e2/go.mod h1:uVDl4OnjvPk07IzoXF/dFM7nBYqAKdJsz4e9xjjWo7Q= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -30,18 +32,28 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= +github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -51,8 +63,12 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/include_test.go b/include_test.go index 344ec8c6..aef6a535 100644 --- a/include_test.go +++ b/include_test.go @@ -11,7 +11,7 @@ func TestIncludeSimple(t *testing.T) { t.Fatal(err) } bals := GetBalances(trans, []string{"Assets"}) - if bals[0].Balance.StringRound() != "50" { + if bals[0].Balance.StringFixed(0) != "50" { t.Fatal(errors.New("should be 50")) } } @@ -22,7 +22,7 @@ func TestIncludeGlob(t *testing.T) { t.Fatal(err) } bals := GetBalances(trans, []string{"Assets"}) - if bals[0].Balance.StringRound() != "80" { + if bals[0].Balance.StringFixed(0) != "80" { t.Fatal(errors.New("should be 80")) } } diff --git a/ledger/camt/camt.go b/ledger/camt/camt.go new file mode 100644 index 00000000..fa26f6ad --- /dev/null +++ b/ledger/camt/camt.go @@ -0,0 +1,91 @@ +package camt + +import ( + "encoding/xml" + "io" +) + +// XML structures for CAMT.053 format +type Document struct { + XMLName xml.Name `xml:"Document"` + BkToCstmrStmt BkToCstmrStmt `xml:"BkToCstmrStmt"` +} + +type BkToCstmrStmt struct { + Stmt Stmt `xml:"Stmt"` +} + +type Stmt struct { + Acct Acct `xml:"Acct"` + Ntry []Ntry `xml:"Ntry"` +} + +type Acct struct { + Id Id `xml:"Id"` + Ccy string `xml:"Ccy"` + Ownr Ownr `xml:"Ownr"` +} + +type Id struct { + IBAN string `xml:"IBAN"` +} + +type Ownr struct { + Nm string `xml:"Nm"` +} + +type Ntry struct { + Amt Amount `xml:"Amt"` + CdtDbtInd string `xml:"CdtDbtInd"` + BookgDt BookgDt `xml:"BookgDt"` + BkTxCd BkTxCd `xml:"BkTxCd"` + NtryRef string `xml:"NtryRef"` + AddtlNtryInf string `xml:"AddtlNtryInf"` + NtryDtls *NtryDtls `xml:"NtryDtls"` +} + +type Amount struct { + Value string `xml:",chardata"` + Ccy string `xml:"Ccy,attr"` +} + +type BookgDt struct { + DtTm string `xml:"DtTm"` +} + +type BkTxCd struct { + Prtry Prtry `xml:"Prtry"` +} + +type Prtry struct { + Cd string `xml:"Cd"` +} + +type NtryDtls struct { + TxDtls TxDtls `xml:"TxDtls"` +} + +type TxDtls struct { + RltdPties RltdPties `xml:"RltdPties"` +} + +type RltdPties struct { + Cdtr *Cdtr `xml:"Cdtr"` +} + +type Cdtr struct { + Pty Pty `xml:"Pty"` +} + +type Pty struct { + Nm string `xml:"Nm"` +} + +func ParseCamt(reader io.Reader) ([]Ntry, error) { + var doc Document + if err := xml.NewDecoder(reader).Decode(&doc); err != nil { + return nil, err + } + + return doc.BkToCstmrStmt.Stmt.Ntry, nil +} diff --git a/ledger/camt/camt_test.go b/ledger/camt/camt_test.go new file mode 100644 index 00000000..602fd98b --- /dev/null +++ b/ledger/camt/camt_test.go @@ -0,0 +1,22 @@ +package camt_test + +import ( + "bytes" + _ "embed" + "testing" + + "github.com/howeyc/ledger/ledger/camt" +) + +//go:embed sample.xml +var camtSample []byte + +func TestParseCamt(t *testing.T) { + entries, err := camt.ParseCamt(bytes.NewBuffer(camtSample)) + if err != nil { + t.Error(err) + } + if len(entries) != 2 { + t.Error("Expected 2 got ", len(entries)) + } +} diff --git a/ledger/camt/sample.xml b/ledger/camt/sample.xml new file mode 100644 index 00000000..11456d1e --- /dev/null +++ b/ledger/camt/sample.xml @@ -0,0 +1,155 @@ + + + + + 1111111-000000 + 2025-07-31T12:37:01.152446900Z + + + 1111111-000000-99999999 + 2025-07-31T12:37:01.152446900Z + + 2025-07-12T00:00:00+01:00 + 2025-07-14T00:00:00+01:00 + + + + BE00000000000 + + EUR + + Sample + + + ADDR + + EU-0000 + Fake + Happy lane + + + + + 0000000001234 + + COID + + + + + + + + Wise Europe SA + + + ADDR + + 1050 + Brussels + Rue du Trône 100, 3rd floor + + + + + + + + CLBD + + + 67.71 + CRDT +
+ 2025-07-14T00:00:00+01:00 +
+ + + + + OPBD + + + 306.61 + CRDT +
+ 2025-07-12T00:00:00+01:00 +
+ + + + + Unrealised gains and losses + + + 2.26 + CRDT +
+ 2025-07-14T00:00:00+01:00 +
+ + + + 2 + -38.90 + + 38.90 + DBIT + + + + 0 + 0 + + + 2 + -38.90 + + + + 3.90 + DBIT + + BOOK + + + 2025-07-13T05:32:45.916737+01:00 + + + + CARD-675 + + + Card transaction of EUR issued + + + 00001/2025 + 35.00 + DBIT + + BOOK + + + 2025-07-12T08:58:01.327701+01:00 + + + + TRANSFER-0000 + + + + + + + + LLC Company + + + + + + Sent money to LLC Company + + + + diff --git a/ledger/cmd/import.go b/ledger/cmd/import.go index da479024..bc5b86e5 100644 --- a/ledger/cmd/import.go +++ b/ledger/cmd/import.go @@ -4,6 +4,7 @@ import ( "encoding/csv" "errors" "fmt" + "log" "math" "os" "strings" @@ -11,10 +12,12 @@ import ( "unicode/utf8" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/import/camt" + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" "github.com/howeyc/ledger/ledger/cmd/internal/import/qfx" + "github.com/howeyc/ledger/ledger/cmd/internal/import/qif" "github.com/jbrukh/bayesian" + "github.com/shopspring/decimal" "github.com/spf13/cobra" ) @@ -27,15 +30,75 @@ var negateAmount bool var allowMatching bool var fieldDelimiter string var scaleFactor float64 +var overrideCurrency string + +type Importer struct { + filename string + reader *os.File + decScale decimal.Decimal + matchingAccount string + generalLedger []*ledger.Transaction + classifier *bayesian.Classifier +} + +func NewImporter(accountSubstring, filename string) *Importer { + imp := Importer{ + filename: filename, + decScale: decimal.NewFromFloat(scaleFactor), + } + + fileReader, err := os.Open(filename) + if err != nil { + fmt.Println("CSV: ", err) + return nil + } + imp.reader = fileReader + + // If a ledger file path is provided, load it and train the classifier. + // Otherwise, skip loading and prediction will fall back to "unknown:unknown". + if ledgerFilePath != "" { + generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) + if parseError != nil { + fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) + return nil + } + imp.generalLedger = generalLedger + + matchingAccount, err := imp.findMatchingAccount(accountSubstring) + if err != nil { + fmt.Println(err) + return nil + } + imp.matchingAccount = matchingAccount -func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string) *bayesian.Classifier { - allAccounts := ledger.GetBalances(generalLedger, []string{}) - classes := make([]bayesian.Class, len(allAccounts)) - for i, bal := range allAccounts { - classes[i] = bayesian.Class(bal.Name) + imp.classifier = imp.trainClassifier(imp.matchingAccount) + } else { + imp.matchingAccount = accountSubstring } + + return &imp +} + +func (imp *Importer) Close() { + imp.reader.Close() +} + +func (imp *Importer) trainClassifier(matchingAccount string) *bayesian.Classifier { + allAccounts := ledger.GetBalances(imp.generalLedger, []string{}) + uniqueAccounts := make(map[string]bool) + for _, acc := range allAccounts { + if ok, _ := uniqueAccounts[acc.Name]; !ok { + uniqueAccounts[acc.Name] = true + } + } + + classes := []bayesian.Class{} + for name := range uniqueAccounts { + classes = append(classes, bayesian.Class(name)) + } + classifier := bayesian.NewClassifier(classes...) - for _, tran := range generalLedger { + for _, tran := range imp.generalLedger { payeeWords := strings.Fields(tran.Payee) // learn accounts names (except matchingAccount) for transactions where matchingAccount is present learnName := false @@ -57,14 +120,18 @@ func trainClassifier(generalLedger []*ledger.Transaction, matchingAccount string return classifier } -func predictAccount(classifier *bayesian.Classifier, inputPayeeWords []string) string { +func (imp *Importer) predictAccount(inputPayeeWords []string) string { + if imp.classifier == nil { + return "unknown:unknown" + } + // Classify into expense account // Find the highest and second highest scores highScore1 := math.Inf(-1) highScore2 := math.Inf(-1) matchIdx := 0 - scores, _, _ := classifier.LogScores(inputPayeeWords) + scores, _, _ := imp.classifier.LogScores(inputPayeeWords) for j, score := range scores { if score > highScore1 { highScore2 = highScore1 @@ -75,15 +142,15 @@ func predictAccount(classifier *bayesian.Classifier, inputPayeeWords []string) s // If the difference between the highest and second highest scores is greater than 10 // then it indicates that highscore is a high confidence match if highScore1-highScore2 > 10 { - return string(classifier.Classes[matchIdx]) + return string(imp.classifier.Classes[matchIdx]) } else { return "unknown:unknown" } } -func findMatchingAccount(generalLedger []*ledger.Transaction, accountSubstring string) (string, error) { +func (imp *Importer) findMatchingAccount(accountSubstring string) (string, error) { var matchingAccount string - matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring}) + matchingAccounts := ledger.GetBalances(imp.generalLedger, []string{accountSubstring}) if len(matchingAccounts) < 1 { return "", ErrNoMatchingAccount } @@ -100,29 +167,8 @@ func findMatchingAccount(generalLedger []*ledger.Transaction, accountSubstring s return matchingAccount, nil } -func importCSV(accountSubstring, csvFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - csvFileReader, err := os.Open(csvFileName) - if err != nil { - fmt.Println("CSV: ", err) - return - } - defer csvFileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - csvReader := csv.NewReader(csvFileReader) +func (imp *Importer) importCSV() { + csvReader := csv.NewReader(imp.reader) csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter) csvRecords, cerr := csvReader.ReadAll() if cerr != nil { @@ -130,8 +176,6 @@ func importCSV(accountSubstring, csvFileName string) { return } - classifier := trainClassifier(generalLedger, matchingAccount) - // Find columns from header var dateColumn, payeeColumn, amountColumn, commentColumn int dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1 @@ -160,12 +204,12 @@ func importCSV(accountSubstring, csvFileName string) { } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + csvAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, record := range csvRecords[1:] { inputPayeeWords := strings.Fields(record[payeeColumn]) csvDate, _ := time.Parse(csvDateFormat, record[dateColumn]) - if allowMatching || !existingTransaction(generalLedger, csvDate, record[payeeColumn]) { - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + if allowMatching || !imp.existingTransaction(csvDate, record[payeeColumn]) { + expenseAccount.Name = imp.predictAccount(inputPayeeWords) // Parse error, set to zero if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil { @@ -180,16 +224,19 @@ func importCSV(accountSubstring, csvFileName string) { } // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Csv amount is the negative of the expense amount csvAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]} trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount} - // Comment + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } if commentColumn >= 0 && record[commentColumn] != "" { trans.Comments = []string{";" + record[commentColumn]} } @@ -198,38 +245,15 @@ func importCSV(accountSubstring, csvFileName string) { } } -func importCamt(accountSubstring, camtFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - fileReader, err := os.Open(camtFileName) - if err != nil { - fmt.Println("CAMT: ", err, camtFileName) - return - } - defer fileReader.Close() - - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return - } - - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) - if err != nil { - fmt.Println(err) - return - } - - classifier := trainClassifier(generalLedger, matchingAccount) - - entries, err := camt.ParseCamt(fileReader) +func (imp *Importer) importCamt() { + entries, err := camt.ParseCamt(imp.reader) if err != nil { fmt.Println("CAMT parse error:", err.Error()) return } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - camtAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + camtAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, entry := range entries { dateTime, err := time.Parse(time.RFC3339, entry.BookgDt.DtTm) if err != nil { @@ -259,7 +283,7 @@ func importCamt(accountSubstring, camtFileName string) { } inputPayeeWords := strings.Fields(payee) - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Name = imp.predictAccount(inputPayeeWords) expenseAccount.Balance = amount // Determine if debit @@ -269,16 +293,22 @@ func importCamt(accountSubstring, camtFileName string) { } // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Csv amount is the negative of the expense amount camtAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: dateTime, Payee: payee} trans.AccountChanges = []ledger.Account{camtAccount, expenseAccount} - - // Comment + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } else if entry.Amt.Ccy != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = entry.Amt.Ccy + } + } if reference != "" { trans.Comments = []string{";" + reference} } @@ -286,38 +316,120 @@ func importCamt(accountSubstring, camtFileName string) { } } -func importQFX(accountSubstring, qfxFileName string) { - decScale := decimal.NewFromFloat(scaleFactor) - - fileReader, err := os.Open(qfxFileName) +func (imp *Importer) importQIF() { + entries, err := qif.ParseQIF(imp.reader) if err != nil { - fmt.Println("QFX: ", err, qfxFileName) + fmt.Println("QIF parse error:", err.Error()) return } - defer fileReader.Close() - generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath) - if parseError != nil { - fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error()) - return + expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} + qifAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} + for _, entry := range entries { + // Parse date (QIF dates are often locale-specific; assume mm/dd/yyyy here) + dateTime, err := time.Parse("01/02/2006", entry.Date) + if err != nil { + // Try an alternate common QIF date format (dd/mm/yyyy) + dateTime, err = time.Parse("02/01/2006", entry.Date) + if err != nil { + fmt.Println("QIF date parse error:", err.Error()) + continue + } + } + + // Parse amount + amount, err := decimal.NewFromString(entry.Amount) + if err != nil { + fmt.Println("QIF amount parse error:", err.Error()) + continue + } + + payee := entry.Payee + inputPayeeWords := strings.Fields(payee) + + expenseAccount.Name = imp.predictAccount(inputPayeeWords) + expenseAccount.Balance = amount + + // Apply scale + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) + + // Account side is the opposite of expense + qifAccount.Balance = expenseAccount.Balance.Neg() + + trans := &ledger.Transaction{Date: dateTime, Payee: payee} + trans.AccountChanges = []ledger.Account{qifAccount, expenseAccount} + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } + if len(entry.RawLines) > 0 { + // Join all raw lines except header/type line + comment := strings.Join(entry.RawLines, " ") + trans.Comments = []string{";" + comment} + } + WriteTransaction(os.Stdout, trans, 80) } +} - matchingAccount, err := findMatchingAccount(generalLedger, accountSubstring) +func (imp *Importer) importIIF() { + f, err := iif.NewDecoder(imp.reader).Decode() if err != nil { - fmt.Println(err) + log.Fatal(err) return } - classifier := trainClassifier(generalLedger, matchingAccount) + tx := []iif.Transaction{} + for _, b := range f.Blocks { + tr, err := iif.DeserializeTransactions(b) + if err != nil { + log.Fatal(err) + return + } + tx = append(tx, tr...) + } + + for _, itx := range tx { + trans := &ledger.Transaction{ + Date: itx.Tr.Date, + Payee: itx.Tr.Class + " " + itx.Tr.Memo, + } + trans.AccountChanges = []ledger.Account{ + { + Name: itx.Tr.Account, + Balance: itx.Tr.Amount, + }, + } + + for _, split := range itx.Splits { + trans.AccountChanges = append( + trans.AccountChanges, + ledger.Account{ + Name: split.Account, + Balance: split.Amount, + }, + ) + } + + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } + WriteTransaction(os.Stdout, trans, 80) + } + +} - entries, err := qfx.ParseQFX(fileReader) +func (imp *Importer) importQFX() { + entries, err := qfx.ParseQFX(imp.reader) if err != nil { fmt.Println("QFX parse error:", err.Error()) return } expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero} - qfxAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero} + qfxAccount := ledger.Account{Name: imp.matchingAccount, Balance: decimal.Zero} for _, entry := range entries { // QFX DTPOSTED is typically YYYYMMDDHHMMSS.XXX; we only care about the date. // Take the first 8 characters as YYYYMMDD. @@ -341,20 +453,22 @@ func importQFX(accountSubstring, qfxFileName string) { payee := entry.Memo inputPayeeWords := strings.Fields(payee) - expenseAccount.Name = predictAccount(classifier, inputPayeeWords) + expenseAccount.Name = imp.predictAccount(inputPayeeWords) expenseAccount.Balance = amount // Apply scale - expenseAccount.Balance = expenseAccount.Balance.Mul(decScale) + expenseAccount.Balance = expenseAccount.Balance.Mul(imp.decScale) // Account side is the opposite of expense qfxAccount.Balance = expenseAccount.Balance.Neg() - // Create valid transaction for print in ledger format trans := &ledger.Transaction{Date: dateTime, Payee: payee} trans.AccountChanges = []ledger.Account{qfxAccount, expenseAccount} - - // Comment with FITID if present + if overrideCurrency != "" { + for i := range trans.AccountChanges { + trans.AccountChanges[i].Currency = overrideCurrency + } + } if entry.FitID != "" { trans.Comments = []string{";" + entry.FitID} } @@ -371,13 +485,21 @@ var importCmd = &cobra.Command{ accountSubstring := args[0] fileName := args[1] + imp := NewImporter(accountSubstring, fileName) + defer imp.Close() + lower := strings.ToLower(fileName) - if strings.HasSuffix(lower, ".xml") { - importCamt(accountSubstring, fileName) - } else if strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx") { - importQFX(accountSubstring, fileName) - } else { - importCSV(accountSubstring, fileName) + switch { + case strings.HasSuffix(lower, ".xml"): + imp.importCamt() + case strings.HasSuffix(lower, ".qfx") || strings.HasSuffix(lower, ".ofx"): + imp.importQFX() + case strings.HasSuffix(lower, ".qif"): + imp.importQIF() + case strings.HasSuffix(lower, ".iif"): + imp.importIIF() + default: + imp.importCSV() } }, @@ -391,11 +513,12 @@ func init() { importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.") importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.") importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.") + importCmd.Flags().StringVar(&overrideCurrency, "override-currency", "", "Override detected currency for imported transactions.") } -func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool { - for _, trans := range generalLedger { - if trans.Date == transDate && strings.TrimSpace(trans.Payee) == strings.TrimSpace(payee) { +func (imp *Importer) existingTransaction(transDate time.Time, payee string) bool { + for _, trans := range imp.generalLedger { + if trans.Date.Equal(transDate) && strings.TrimSpace(trans.Payee) == strings.TrimSpace(payee) { return true } } diff --git a/ledger/cmd/import_test.go b/ledger/cmd/import_test.go index b78105cc..8ce2bc87 100644 --- a/ledger/cmd/import_test.go +++ b/ledger/cmd/import_test.go @@ -32,7 +32,8 @@ func Test_findMatchingAccount(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, gotErr := findMatchingAccount(tt.generalLedger, tt.accountSubstring) + imp := &Importer{generalLedger: tt.generalLedger} + got, gotErr := imp.findMatchingAccount(tt.accountSubstring) if gotErr != nil { if !tt.wantErr { t.Errorf("findMatchingAccount() failed: %v", gotErr) diff --git a/ledger/cmd/internal/httpcompress/compress.go b/ledger/cmd/internal/httpcompress/compress.go index 8ef43d98..a129d2f0 100644 --- a/ledger/cmd/internal/httpcompress/compress.go +++ b/ledger/cmd/internal/httpcompress/compress.go @@ -16,9 +16,9 @@ type CompressResponseWriter struct { } func (res CompressResponseWriter) Write(b []byte) (int, error) { - if "" == res.Header().Get("Content-Type") { + if "" == res.ResponseWriter.Header().Get("Content-Type") { // If no content type, apply sniffing algorithm to un-gzipped body. - res.Header().Set("Content-Type", http.DetectContentType(b)) + res.ResponseWriter.Header().Set("Content-Type", http.DetectContentType(b)) } return res.Writer.Write(b) } diff --git a/ledger/cmd/internal/import/iif/Full Bill payment.iif b/ledger/cmd/internal/import/iif/Full Bill payment.iif new file mode 100644 index 00000000..a90d6291 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Bill payment.iif @@ -0,0 +1,11 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Accounts Payable AP 2000 +!VEND NAME REFNUM PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Vendor 1 Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM MEMO CLEAR TOPRINT +!SPL SPLID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM MEMO CLEAR QNTY +!ENDTRNS +TRNS BILLPMT 7/16/1998 Checking Vendor -35 Test Memo N Y +SPL BILLPMT 7/16/1998 Accounts Payable Vendor 35 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Deposit.iif b/ledger/cmd/internal/import/iif/Full Deposit.iif new file mode 100644 index 00000000..cd01b6f1 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Deposit.iif @@ -0,0 +1,15 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Income INC +!CLASS NAME +CLASS class +!CUST NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 SADDR1 SADDR2 SADDR3 SADDR4 SADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 CTYPE TERMS TAXABLE LIMIT RESALENUM REP TAXITEM NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +CUST Customer Joe Customer 444 Road Rd "Anywhere, AZ 85740" USA 5554443333 Joe Customer N Joe Customer +!OTHERNAME NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +OTHERNAME Other Name Other Name 123 a Street "Somewhere, AZ 85730" USA 5555555555 Other Name Other Name +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR +!SPL SPLID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR +!ENDTRNS +TRNS DEPOSIT 7/1/1998 Checking 10000 N +SPL DEPOSIT 7/1/1998 Income Customer -10000 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Invoice.iif b/ledger/cmd/internal/import/iif/Full Invoice.iif new file mode 100644 index 00000000..28251191 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Invoice.iif @@ -0,0 +1,24 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Accounts Receivable AR 1200 +ACCNT Construction:Labor INC 4100 +ACCNT Construction:Materials INC 4200 +ACCNT Inventory Asset OCASSET 1120 INVENTORYASSET +ACCNT Cost of Goods Sold COGS Cost of Goods Sold 5000 COGS +!INVITEM NAME INVITEMTYPE DESC PURCHASEDESC ACCNT ASSETACCNT COGSACCNT PRICE COST TAXABLE PAYMETH TAXVEND TAXDIST PREFVEND REORDERPOINT EXTRA +INVITEM Framing SERV Framing labor Construction:Labor 55 0 N +INVITEM Wood Door:Exterior INVENTORY Exterior wood door Exterior door - #P-10981 Construction:Materials Inventory Asset Cost of Goods Sold 120 105 Y Perry Windows & Doors 5 +INVITEM Hardware:Doorknobs Std INVENTORY Standard Doorknobs Doorknobs Part # DK 3704 Construction:Materials Inventory Asset Cost of Goods Sold 30 27 Y Patton Hardware Supplies 50 +!CLASS NAME +CLASS class +!CUST NAME BADDR1 BADDR2 BADDR3 BADDR4 BADDR5 SADDR1 SADDR2 SADDR3 SADDR4 SADDR5 PHONE1 PHONE2 FAXNUM EMAIL NOTE CONT1 CONT2 CTYPE TERMS TAXABLE LIMIT RESALENUM REP TAXITEM NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +CUST Customer Joe Customer 444 Road Rd "Anywhere, AZ 85740" USA 5554443333 Joe Customer N Joe Customer +!VEND NAME PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Vendor Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR TOPRINT NAMEISTAXABLE ADDR1 ADDR3 TERMS SHIPVIA SHIPDATE +!SPL SPLID TRNSTYPE DATE ACCNT NAME CLASS AMOUNT DOCNUM MEMO CLEAR QNTY PRICE INVITEM TAXABLE OTHER2 YEARTODATE WAGEBASE +!ENDTRNS +TRNS INVOICE 7/18/98 Accounts Receivable Customer 205 1 N Y N 7/16/98 +SPL INVOICE 7/16/98 Construction:Labor -55 Framing labor N 55 Framing N 0 0 +SPL INVOICE 7/16/98 Construction:Materials -120 Exterior wood door N 120 Wood Door:Exterior Y 0 0 +SPL INVOICE 7/16/98 Construction:Materials -30 Standard Doorknobs N 30 Hardware:Doorknobs Std Y 0 0 +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif b/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif new file mode 100644 index 00000000..9d22a5c6 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Sales Tax Payment.iif @@ -0,0 +1,13 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Sales Tax Payable OCLIAB 2200 SALESTAX +!INVITEM NAME INVITEMTYPE DESC PURCHASEDESC ACCNT ASSETACCNT COGSACCNT PRICE COST TAXABLE PAYMETH TAXVEND TAXDIST PREFVEND REORDERPOINT EXTRA +INVITEM San Domingo COMPTAX "CA sales tax, San Domingo County" Sales Tax Payable 7.50% 0 N Sales Tax Vendor +!VEND NAME PRINTAS ADDR1 ADDR2 ADDR3 ADDR4 ADDR5 VTYPE CONT1 CONT2 PHONE1 PHONE2 FAXNUM EMAIL NOTE TAXID LIMIT TERMS NOTEPAD SALUTATION COMPANYNAME FIRSTNAME MIDINIT LASTNAME +VEND Sales Tax Vendor Jon Vendor 555 Street St "Anywhere, AZ 85730" USA Jon Vendor 5555555555 Jon Vendor +!TRNS TRNSID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM CLEAR TOPRINT ADDR1 +!SPL SPLID TRNSTYPE DATE ACCNT NAME AMOUNT DOCNUM CLEAR QNTY INVITEM +!ENDTRNS +TRNS SALESTAXPMT 8/5/98 Checking Sales Tax Vendor -66.57 N Y Jon Vendor +SPL SALESTAXPMT 8/5/98 Sales Tax Payable Sales Tax Vendor 66.57 N San Domingo +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/Full Transfer.iif b/ledger/cmd/internal/import/iif/Full Transfer.iif new file mode 100644 index 00000000..1b019524 --- /dev/null +++ b/ledger/cmd/internal/import/iif/Full Transfer.iif @@ -0,0 +1,9 @@ +!ACCNT NAME ACCNTTYPE DESC ACCNUM EXTRA +ACCNT Checking BANK +ACCNT Savings BANK +!TRNS TRNSID TRNSTYPE DATE ACCNT AMOUNT MEMO CLEAR +!SPL SPLID TRNSTYPE DATE ACCNT AMOUNT MEMO CLEAR +!ENDTRNS +TRNS TRANSFER 7/1/98 Checking -500 Funds Transfer N +SPL TRANSFER 7/1/98 Savings 500 N +ENDTRNS diff --git a/ledger/cmd/internal/import/iif/iif.go b/ledger/cmd/internal/import/iif/iif.go new file mode 100644 index 00000000..92d41e77 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif.go @@ -0,0 +1,188 @@ +package iif + +import ( + "encoding/csv" + "errors" + "io" + "strings" +) + +var ( + ErrInvalidHeaderLine = errors.New("iif: invalid header line") + ErrMismatchedColumns = errors.New("iif: mismatched number of columns") + ErrMismatchedRecords = errors.New("iif: row does not match expected header") + ErrUnknownRecordType = errors.New("iif: unknown record type") + ErrUnexpectedSectionType = errors.New("iif: unexpected record type for current section") + ErrEmptyHeader = errors.New("iif: empty header") +) + +type RecordType string + +type Header struct { + Type RecordType + Fields []string +} + +type Record struct { + Type RecordType + Fields map[string]string +} + +type Block struct { + Records [][]Record + Headers []Header +} + +type File struct { + Blocks []Block +} + +type Decoder struct { + r *csv.Reader + err error + IsHeader bool + Type RecordType + Fields []string +} + +func NewDecoder(r io.Reader) *Decoder { + reader := csv.NewReader(r) + reader.Comma = '\t' + reader.LazyQuotes = true + reader.TrimLeadingSpace = false + reader.FieldsPerRecord = -1 + d := Decoder{r: reader} + d.Next() + return &d +} + +func (d *Decoder) Next() { + line, err := d.r.Read() + d.err = err + if err == nil { + d.IsHeader = strings.HasPrefix(line[0], "!") + if d.IsHeader { + d.Type = RecordType(line[0][1:]) + } else { + d.Type = RecordType(line[0]) + } + d.Fields = line[1:] + } +} + +func (d *Decoder) Error() error { + if d.err != io.EOF { + return d.err + } + return nil +} + +func (d *Decoder) Done() bool { + return d.err != nil +} + +func (f *File) Load(d *Decoder) error { + for !d.Done() { + if d.Error() != nil { + return d.Error() + } + b := Block{} + err := b.Load(d) + if err != nil { + return err + } + f.Blocks = append(f.Blocks, b) + } + return nil +} + +func (h Header) MapFields(fields []string) map[string]string { + m := make(map[string]string, len(fields)) + for i, f := range h.Fields { + if i >= len(fields) { + break + } + m[f] = fields[i] + } + return m +} + +func (b *Block) Load(d *Decoder) error { + if d.Done() { + return d.Error() + } + // Parse Headers + for !d.Done() && d.IsHeader { + b.Headers = append( + b.Headers, + Header{ + Type: RecordType(d.Type), + Fields: trimLine(d.Fields), + }, + ) + d.Next() + } + if d.Error() != nil { + return d.Error() + } + + // Parse Records + for !d.Done() && !d.IsHeader { + r := []Record{} + // At least one record per header + if len(b.Headers) == 0 { + return ErrEmptyHeader + } + for _, h := range b.Headers { + if d.Done() { + return d.Error() + } + if d.Done() || d.Type != h.Type { + return ErrMismatchedRecords + } + + for !d.Done() && !d.IsHeader && d.Type == h.Type { + r = append(r, Record{ + Type: d.Type, + Fields: h.MapFields(d.Fields), + }) + d.Next() + } + if len(r) == 0 { + return ErrMismatchedRecords + } + } + b.Records = append(b.Records, r) + } + return nil +} + +func trimLine(records []string) []string { + for i, r := range records { + if r == "" { + return records[:i] + } + } + return records +} + +func (d *Decoder) Decode() (*File, error) { + f := File{} + err := f.Load(d) + if err != nil && err != io.EOF { + return nil, err + } + return &f, nil +} + +type Encoder struct { + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(f *File) error { + return errors.New("iif encoding not implemented") +} diff --git a/ledger/cmd/internal/import/iif/iif_test.go b/ledger/cmd/internal/import/iif/iif_test.go new file mode 100644 index 00000000..aecda940 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_test.go @@ -0,0 +1,151 @@ +package iif_test + +import ( + "bytes" + "reflect" + "testing" + + _ "embed" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" +) + +var ( + //go:embed "Full Deposit.iif" + fullDepositIIF []byte + + //go:embed "Full Invoice.iif" + fullInvoiceIIF []byte + + //go:embed "Full Bill payment.iif" + fullBillPaymentIIF []byte + + //go:embed "Full Sales Tax Payment.iif" + fullSalesTaxPaymentIIF []byte + + //go:embed "Full Transfer.iif" + fullTransferIIF []byte +) + +func TestDecodeEncode(t *testing.T) { + tests := []struct { + name string + data []byte + blocks []iif.Block + }{ + { + name: "fullDepositIIF", + data: fullDepositIIF, + blocks: []iif.Block{ + { + Headers: []iif.Header{ + {Type: iif.RecordType("ACCNT"), Fields: []string{"NAME", "ACCNTTYPE", "DESC", "ACCNUM", "EXTRA"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("CLASS"), Fields: []string{"NAME"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("CUST"), Fields: []string{"NAME", "BADDR1", "BADDR2", "BADDR3", "BADDR4", "BADDR5", "SADDR1"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("OTHERNAME"), Fields: []string{"NAME", "BADDR1", "BADDR2", "BADDR3", "BADDR4", "BADDR5", "PHONE1", "PHONE2", "FAXNUM", "EMAIL", "NOTE", "CONT1", "CONT2", "NOTEPAD", "SALUTATION", "COMPANYNAME", "FIRSTNAME", "MIDINIT", "LASTNAME"}}, + }, + }, + { + Headers: []iif.Header{ + {Type: iif.RecordType("TRNS"), Fields: []string{"TRNSID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("SPL"), Fields: []string{"SPLID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("ENDTRNS"), Fields: []string{}}, + }, + Records: [][]iif.Record{ + { + { + Type: iif.RecordType("TRNS"), + Fields: map[string]string{ + "TRNSID": " ", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Checking", + "NAME": "", + "CLASS": "", + "AMOUNT": "10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("SPL"), + Fields: map[string]string{ + "SPLID": "", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Income", + "NAME": "Customer", + "CLASS": "", + "AMOUNT": "-10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("ENDTRNS"), + Fields: map[string]string{}, + }, + }, + }, + }, + }, + }, + { + name: "fullInvoiceIIF", + data: fullInvoiceIIF, + }, + { + name: "fullBillPaymentIIF", + data: fullBillPaymentIIF, + }, + { + name: "fullSalesTaxPaymentIIF", + data: fullSalesTaxPaymentIIF, + }, + { + name: "fullTransferIIF", + data: fullTransferIIF, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dec := iif.NewDecoder(bytes.NewReader(tt.data)) + f, err := dec.Decode() + if err != nil { + t.Fatalf("Decode error: %v", err) + } + + if len(f.Blocks) == 0 { + t.Error("missing blocks from file") + } + + for i, b := range tt.blocks { + if i >= len(f.Blocks) { + t.Errorf("expected at least %d blocks, got %d", len(tt.blocks), len(f.Blocks)) + break + } + if !reflect.DeepEqual(b.Headers, f.Blocks[i].Headers) { + t.Errorf("expected headers to equal %+v != %+v", b.Headers, f.Blocks[i].Headers) + } + if b.Records != nil && !reflect.DeepEqual(b.Records, f.Blocks[i].Records) { + t.Errorf("expected records to equal %+v != %+v", b.Records, f.Blocks[i].Records) + } + } + }) + } +} diff --git a/ledger/cmd/internal/import/iif/iif_trns.go b/ledger/cmd/internal/import/iif/iif_trns.go new file mode 100644 index 00000000..ef79a558 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_trns.go @@ -0,0 +1,167 @@ +package iif + +import ( + "fmt" + "reflect" + "time" + + "github.com/shopspring/decimal" +) + +type Transaction struct { + Tr Trns `type:"TRNS"` + Splits []Spl `type:"SPL"` +} + +type Trns struct { + TransactionType string `iif:"TRNSTYPE"` + Date time.Time `iif:"DATE"` + Account string `iif:"ACCNT"` + Name string `iif:"NAME"` + Class string `iif:"CLASS"` + Amount decimal.Decimal `iif:"AMOUNT"` + Memo string `iif:"MEMO"` +} + +type Spl struct { + TransactionType string `iif:"TRNSTYPE"` + Date time.Time `iif:"DATE"` + Account string `iif:"ACCNT"` + Name string `iif:"NAME"` + Class string `iif:"CLASS"` + Amount decimal.Decimal `iif:"AMOUNT"` + Memo string `iif:"MEMO"` +} + +func DeserializeTransactions(b Block) ([]Transaction, error) { + var out []Transaction + + for _, recGroup := range b.Records { + if len(recGroup) == 0 { + continue + } + + var tx Transaction + if err := DeserializeRecordGroup(&tx, recGroup); err != nil { + return nil, err + } + out = append(out, tx) + } + + return out, nil +} + +func DeserializeRecordGroup(tx any, recs []Record) error { + for _, r := range recs { + if err := applyRecord(tx, r); err != nil { + return err + } + } + return nil +} + +func applyRecord(tx any, r Record) error { + txVal := reflect.ValueOf(tx).Elem() + txType := txVal.Type() + + for i := 0; i < txType.NumField(); i++ { + field := txType.Field(i) + tag := field.Tag.Get("type") + if tag == "" || string(r.Type) != tag { + continue + } + + fv := txVal.Field(i) + + if fv.Kind() == reflect.Slice { + elemType := fv.Type().Elem() + elemPtr := reflect.New(elemType).Elem() + + if err := populateStructFromRecord(elemPtr, r); err != nil { + return err + } + + fv.Set(reflect.Append(fv, elemPtr)) + return nil + } + if fv.Kind() == reflect.Struct { + if err := populateStructFromRecord(fv, r); err != nil { + return err + } + return nil + } + } + return nil +} + +func populateStructFromRecord(v reflect.Value, r Record) error { + if v.Kind() != reflect.Struct { + return fmt.Errorf("populateStructFromRecord: expected struct, got %s", v.Kind()) + } + + t := v.Type() + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + tag := sf.Tag.Get("iif") + if tag == "" { + continue + } + + raw, ok := r.Fields[tag] + if !ok { + continue + } + + fv := v.Field(i) + if !fv.CanSet() { + continue + } + + if err := setFieldValueFromString(fv, raw); err != nil { + return fmt.Errorf("field %s: %w", sf.Name, err) + } + } + + return nil +} + +// setFieldValueFromString converts the string representation from a Record +// into the appropriate Go type and assigns it to fv. +func setFieldValueFromString(fv reflect.Value, s string) error { + switch fv.Kind() { + case reflect.String: + fv.SetString(s) + return nil + case reflect.Struct: + // Handle known struct types (time.Time, decimal.Decimal, etc.) + switch fv.Type() { + case reflect.TypeOf(time.Time{}): + // IIF date formats can vary; here we assume the standard + // QuickBooks IIF date "MM/DD/YYYY". Adjust if needed. + if s == "" { + return nil + } + t, err := time.Parse("1/2/2006", s) + if err != nil { + return err + } + fv.Set(reflect.ValueOf(t)) + return nil + case reflect.TypeOf(decimal.Decimal{}): + if s == "" { + fv.Set(reflect.ValueOf(decimal.Zero)) + return nil + } + d, err := decimal.NewFromString(s) + if err != nil { + return err + } + fv.Set(reflect.ValueOf(d)) + return nil + default: + return fmt.Errorf("unsupported struct type %s", fv.Type()) + } + default: + return fmt.Errorf("unsupported kind %s", fv.Kind()) + } +} diff --git a/ledger/cmd/internal/import/iif/iif_trns_test.go b/ledger/cmd/internal/import/iif/iif_trns_test.go new file mode 100644 index 00000000..aeee2c85 --- /dev/null +++ b/ledger/cmd/internal/import/iif/iif_trns_test.go @@ -0,0 +1,115 @@ +package iif_test + +import ( + "reflect" + "testing" + "time" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/iif" + "github.com/shopspring/decimal" +) + +func TestDeserializeTransactions(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + b iif.Block + want []iif.Transaction + wantErr bool + }{ + { + name: "empty", + b: iif.Block{ + Headers: []iif.Header{ + {Type: iif.RecordType("ACCNT"), Fields: []string{"NAME", "ACCNTTYPE", "DESC", "ACCNUM", "EXTRA"}}, + }, + }, + want: nil, + }, + { + name: "simple", + b: iif.Block{ + Headers: []iif.Header{ + {Type: iif.RecordType("TRNS"), Fields: []string{"TRNSID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("SPL"), Fields: []string{"SPLID", "TRNSTYPE", "DATE", "ACCNT", "NAME", "CLASS", "AMOUNT", "DOCNUM", "MEMO", "CLEAR"}}, + {Type: iif.RecordType("ENDTRNS"), Fields: []string{}}, + }, + Records: [][]iif.Record{ + { + { + Type: iif.RecordType("TRNS"), + Fields: map[string]string{ + "TRNSID": " ", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Checking", + "NAME": "", + "CLASS": "", + "AMOUNT": "10000", + "DOCNUM": "", + "MEMO": "Hello", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("SPL"), + Fields: map[string]string{ + "SPLID": "", + "TRNSTYPE": "DEPOSIT", + "DATE": "7/1/1998", + "ACCNT": "Income", + "NAME": "Customer", + "CLASS": "", + "AMOUNT": "-10000", + "DOCNUM": "", + "MEMO": "", + "CLEAR": "N", + }, + }, + { + Type: iif.RecordType("ENDTRNS"), + Fields: map[string]string{}, + }, + }, + }, + }, + want: []iif.Transaction{ + { + Tr: iif.Trns{ + TransactionType: "DEPOSIT", + Date: time.Date(1998, 7, 1, 0, 0, 0, 0, time.UTC), + Account: "Checking", + Amount: decimal.NewFromInt(10000), + Memo: "Hello", + }, + Splits: []iif.Spl{ + { + TransactionType: "DEPOSIT", + Date: time.Date(1998, 7, 1, 0, 0, 0, 0, time.UTC), + Account: "Income", + Name: "Customer", + Amount: decimal.NewFromInt(-10000), + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := iif.DeserializeTransactions(tt.b) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("DeserializeTransactions() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("DeserializeTransactions() succeeded unexpectedly") + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DeserializeTransactions() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/ledger/cmd/internal/import/qif/qif.go b/ledger/cmd/internal/import/qif/qif.go new file mode 100644 index 00000000..ec855a28 --- /dev/null +++ b/ledger/cmd/internal/import/qif/qif.go @@ -0,0 +1,200 @@ +package qif + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// Non-investment QIF transaction, based on the "Non-investment transaction format" +// from the GnuCash documentation. Only a subset of fields is modeled for now. +type Transaction struct { + // Header/type line, e.g. "!Type:Cash" + Type string `qif:"header"` + + // Core transaction fields + Date string `qif:"D"` // D - Date + Amount string `qif:"T"` // T - Amount + Num string `qif:"N"` // N - Number (check/reference) + Payee string `qif:"P"` // P - Payee/description + Memo string `qif:"M"` // M - Memo + Addr string `qif:"A"` // A - Address (multi-line; kept concatenated with '\n') + Cleared string `qif:"C"` // C - Cleared status + Category string `qif:"L"` // L - Category (or transfer/class) + + // Split fields – repeated groups, flattened for now to first occurrence + SplitCategory string `qif:"S"` // S - Category in split + SplitMemo string `qif:"E"` // E - Memo in split + SplitAmount string `qif:"$"` // $ - Dollar amount of split + + // RawLines contains the raw QIF lines (without trailing newline) that + // composed this transaction, excluding the header and trailing '^'. + RawLines []string `qif:"-"` +} + +// Decoder reads QIF data from an input stream. +type Decoder struct { + r *bufio.Reader +} + +// NewDecoder returns a new QIF decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + r: bufio.NewReader(r), + } +} + +// Decode reads QIF data from the underlying reader and returns all parsed +// non-investment transactions. For now this is a convenience wrapper around +// a streaming decode; it reads the whole file. +func (d *Decoder) Decode() ([]*Transaction, error) { + var ( + transactions []*Transaction + currentType string + ) + + for { + line, err := d.readLine() + if err == io.EOF { + // No partial transaction handling – QIF files should end with '^' + return transactions, nil + } + if err != nil { + return nil, err + } + + if len(line) == 0 { + continue + } + + // Header / account-type line: !Type:Cash, !Type:Bank, ... + if strings.HasPrefix(line, "!Type:") { + currentType = strings.TrimSpace(line[len("!Type:"):]) + continue + } + + // A transaction must start with 'D' (date) according to the spec. + if line[0] == 'D' { + tx, err := d.decodeTransaction(currentType, line) + if err != nil { + return nil, err + } + transactions = append(transactions, tx) + continue + } + + // Lines outside of transactions are currently ignored. + } + +} + +// decodeTransaction parses a single transaction, given that the first line +// (already read) is a 'D' date line. It continues reading until the '^' end +// marker has been consumed. +func (d *Decoder) decodeTransaction(txType string, firstLine string) (*Transaction, error) { + tx := &Transaction{ + Type: txType, + } + + assignField(tx, firstLine) + + for { + line, err := d.readLine() + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("unexpected EOF while reading transaction") + } + return nil, err + } + if len(line) == 0 { + // empty lines inside a transaction are preserved in RawLines but + // don't correspond to any field. + tx.RawLines = append(tx.RawLines, line) + continue + } + if line[0] == '^' { + // end of transaction + return tx, nil + } + + assignField(tx, line) + } +} + +// assignField updates tx based on a single QIF field line. +// It also appends the raw line (minus trailing newline) to RawLines. +func assignField(tx *Transaction, line string) { + if len(line) == 0 { + return + } + // Store raw line + tx.RawLines = append(tx.RawLines, line) + + prefix := line[0] + value := line[1:] + + switch prefix { + case 'D': + tx.Date = value + case 'T': + tx.Amount = value + case 'U': + // Higher precision amount; if present, prefer it over T. + tx.Amount = value + case 'N': + tx.Num = value + case 'P': + tx.Payee = value + case 'M': + if tx.Memo == "" { + tx.Memo = value + } else { + // Multiple memo lines – concatenate with newline. + tx.Memo += "\n" + value + } + case 'A': + if tx.Addr == "" { + tx.Addr = value + } else { + tx.Addr += "\n" + value + } + case 'C': + tx.Cleared = value + case 'L': + tx.Category = value + case 'S': + // For now we keep only first split; real-world usage may need a slice. + if tx.SplitCategory == "" { + tx.SplitCategory = value + } + case 'E': + if tx.SplitMemo == "" { + tx.SplitMemo = value + } + case '$': + if tx.SplitAmount == "" { + tx.SplitAmount = value + } + } +} + +// readLine reads a single logical line without the trailing '\n' or '\r\n'. +func (d *Decoder) readLine() (string, error) { + line, err := d.r.ReadString('\n') + if err != nil && err != io.EOF { + return "", err + } + // Trim CRLF and LF. + line = strings.TrimRight(line, "\r\n") + if err == io.EOF && len(line) == 0 { + return "", io.EOF + } + return line, err +} + +// ParseQIF is a convenience helper that parses all transactions from a QIF +// stream and returns them. +func ParseQIF(reader io.Reader) ([]*Transaction, error) { + return NewDecoder(reader).Decode() +} diff --git a/ledger/cmd/internal/import/qif/qif_test.go b/ledger/cmd/internal/import/qif/qif_test.go new file mode 100644 index 00000000..224e8580 --- /dev/null +++ b/ledger/cmd/internal/import/qif/qif_test.go @@ -0,0 +1,101 @@ +package qif_test + +import ( + "bytes" + _ "embed" + "testing" + + "github.com/howeyc/ledger/ledger/cmd/internal/import/qif" +) + +//go:embed sample.qif +var qifSample []byte + +func TestParseQIF(t *testing.T) { + entries, err := qif.ParseQIF(bytes.NewBuffer(qifSample)) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 3 { + t.Fatalf("Expected 3 entries, got %d", len(entries)) + } + + tests := []struct { + index int + typ string + date string + amount string + payee string + memo string + cat string + splitCt string + splitAm string + }{ + { + index: 0, + typ: "Cash", + date: "08/14/2024", + amount: "15.00", + payee: "", + memo: "~@~CLD:1723446000~@~", + cat: "Bank Deposit to PP Account ", + splitCt: "Bank Deposit to PP Account ", + splitAm: "15.00", + }, + { + index: 1, + typ: "Cash", + date: "08/14/2024", + amount: "-15.00", + payee: "9171-5573 Quebec Inc", + memo: "VOIPMS15", + cat: "PreApproved Payment Bill User Payment", + splitCt: "PreApproved Payment Bill User Payment", + splitAm: "-15.00", + }, + { + index: 2, + typ: "Cash", + date: "08/27/2024", + amount: "80.00", + payee: "", + memo: "", + cat: "Bank Deposit to PP Account ", + splitCt: "Bank Deposit to PP Account ", + splitAm: "80.00", + }, + } + + for _, tt := range tests { + if tt.index >= len(entries) { + t.Fatalf("test index %d out of range, len(entries)=%d", tt.index, len(entries)) + } + e := entries[tt.index] + + if e.Type != tt.typ { + t.Errorf("entry %d: expected Type %q, got %q", tt.index, tt.typ, e.Type) + } + if e.Date != tt.date { + t.Errorf("entry %d: expected Date %q, got %q", tt.index, tt.date, e.Date) + } + if e.Amount != tt.amount { + t.Errorf("entry %d: expected Amount %q, got %q", tt.index, tt.amount, e.Amount) + } + if e.Payee != tt.payee { + t.Errorf("entry %d: expected Payee %q, got %q", tt.index, tt.payee, e.Payee) + } + if e.Memo != tt.memo { + t.Errorf("entry %d: expected Memo %q, got %q", tt.index, tt.memo, e.Memo) + } + if e.Category != tt.cat { + t.Errorf("entry %d: expected Category %q, got %q", tt.index, tt.cat, e.Category) + } + if e.SplitCategory != tt.splitCt { + t.Errorf("entry %d: expected SplitCategory %q, got %q", tt.index, tt.splitCt, e.SplitCategory) + } + if e.SplitAmount != tt.splitAm { + t.Errorf("entry %d: expected SplitAmount %q, got %q", tt.index, tt.splitAm, e.SplitAmount) + } + } +} diff --git a/ledger/cmd/internal/import/qif/sample.qif b/ledger/cmd/internal/import/qif/sample.qif new file mode 100644 index 00000000..0c35604e --- /dev/null +++ b/ledger/cmd/internal/import/qif/sample.qif @@ -0,0 +1,35 @@ +!Type:Cash +D08/14/2024 +T15.00 +LBank Deposit to PP Account +SBank Deposit to PP Account +$15.00 +SFee +$0.00 +CX +M~@~CLD:1723446000~@~ +P +^ +!Type:Cash +D08/14/2024 +T-15.00 +LPreApproved Payment Bill User Payment +SPreApproved Payment Bill User Payment +$-15.00 +SFee +$0.00 +CX +MVOIPMS15 +P9171-5573 Quebec Inc +^ +!Type:Cash +D08/27/2024 +T80.00 +LBank Deposit to PP Account +SBank Deposit to PP Account +$80.00 +SFee +$0.00 +CX +P +^ diff --git a/ledger/cmd/print.go b/ledger/cmd/print.go index 70f4b43f..35e08fe4 100644 --- a/ledger/cmd/print.go +++ b/ledger/cmd/print.go @@ -13,10 +13,10 @@ import ( "time" "unicode/utf8" + "github.com/araddon/dateparse" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/fastcolor" - date "github.com/joyt/godate" + "github.com/shopspring/decimal" "github.com/spf13/cobra" "golang.org/x/term" ) @@ -46,8 +46,8 @@ func cliTransactions() ([]*ledger.Transaction, error) { } } - parsedStartDate, tstartErr := date.Parse(startString) - parsedEndDate, tendErr := date.Parse(endString) + parsedStartDate, tstartErr := dateparse.ParseAny(startString) + parsedEndDate, tendErr := dateparse.ParseAny(endString) if tstartErr != nil || tendErr != nil { return nil, errors.New("unable to parse start or end date string argument") @@ -59,7 +59,7 @@ func cliTransactions() ([]*ledger.Transaction, error) { var generalLedger []*ledger.Transaction var parseError error if ledgerFilePath == "-" { - generalLedger, parseError = ledger.ParseLedger(os.Stdin) + generalLedger, parseError = ledger.ParseLedger("", os.Stdin) } else { generalLedger, parseError = ledger.ParseLedgerFile(ledgerFilePath) } @@ -133,7 +133,7 @@ func PrintBalances(accountList []*ledger.Account, printZeroBalances bool, depth, overallBalance = overallBalance.Add(account.Balance) } if (printZeroBalances || account.Balance.Sign() != 0) && (depth < 0 || accDepth <= depth) { - outBalanceString := account.Balance.StringFixedBank() + outBalanceString := account.Currency + " " + account.Balance.StringFixedBank(2) amtColor := colorReset if account.Balance.Sign() < 0 { amtColor = colorNeg @@ -145,7 +145,7 @@ func PrintBalances(accountList []*ledger.Account, printZeroBalances bool, depth, } } fmt.Fprintln(buf, strings.Repeat("-", columns)) - outBalanceString := overallBalance.StringFixedBank() + outBalanceString := overallBalance.StringFixedBank(2) amtColor := colorReset if overallBalance.Sign() < 0 { amtColor = colorNeg @@ -186,7 +186,16 @@ func WriteTransaction(w io.StringWriter, trans *ledger.Transaction, columns int) } w.WriteString(newLine) for _, accChange := range trans.AccountChanges { - outBalanceString := accChange.Balance.StringFixedBank() + outBalanceString := accChange.Balance.StringFixedBank(2) + if accChange.Currency != "" { + outBalanceString = accChange.Currency + " " + outBalanceString + } + // Show converted amount (@@) or conversion factor (@) similar to hledger + if accChange.Converted != nil { + outBalanceString = outBalanceString + " @@ " + accChange.Converted.StringFixedBank(2) + } else if accChange.ConversionFactor != nil { + outBalanceString = outBalanceString + " @ " + accChange.ConversionFactor.String() + } spaceCount := columns - 4 - utf8.RuneCountInString(accChange.Name) - utf8.RuneCountInString(outBalanceString) if spaceCount < 1 { spaceCount = 1 @@ -242,7 +251,9 @@ func PrintRegister(generalLedger []*ledger.Transaction, filterArr []string, colu colorReset := fastcolor.Reset buf := bufio.NewWriter(os.Stdout) - runningBalance := decimal.Zero + // runningBalance keeps the total per currency + runningBalance := make(map[string]decimal.Decimal) + for _, trans := range generalLedger { for _, accChange := range trans.AccountChanges { inFilter := len(filterArr) == 0 @@ -251,30 +262,104 @@ func PrintRegister(generalLedger []*ledger.Transaction, filterArr []string, colu inFilter = true } } - if inFilter { - runningBalance = runningBalance.Add(accChange.Balance) - outBalanceString := accChange.Balance.StringFixedBank() - outRunningBalanceString := runningBalance.StringFixedBank() + if !inFilter { + continue + } - balamtColor := colorReset - if accChange.Balance.Sign() < 0 { - balamtColor = colorNeg + // Update running totals per currency + cur := accChange.Currency + if cur == "" { + cur = "_" // treat empty currency as its own bucket + } + runningBalance[cur] = runningBalance[cur].Add(accChange.Balance) + + // Current posting amount string + outBalanceString := accChange.Balance.StringFixedBank(2) + if accChange.Currency != "" { + outBalanceString = accChange.Currency + " " + outBalanceString + } + + // Build primary running total string (first currency: the one for this posting) + type curTotal struct { + currency string + amount decimal.Decimal + } + totals := make([]curTotal, 0, len(runningBalance)) + for k, v := range runningBalance { + totals = append(totals, curTotal{currency: k, amount: v}) + } + // Sort for deterministic output: primary currency first, then by name + slices.SortFunc(totals, func(a, b curTotal) int { + // primary currency first + if a.currency == cur && b.currency != cur { + return -1 + } + if b.currency == cur && a.currency != cur { + return 1 + } + // "_" (no currency) should sort last + if a.currency == "_" && b.currency != "_" { + return 1 } - runamtColor := colorReset - if runningBalance.Sign() < 0 { - runamtColor = colorNeg + if b.currency == "_" && a.currency != "_" { + return -1 } + return strings.Compare(a.currency, b.currency) + }) - buf.WriteString(trans.Date.Format(transactionDateFormat)) - buf.WriteString(" ") - colorPayee.WriteStringFixed(buf, trans.Payee, col1width, false) - buf.WriteString(" ") - colorAccount.WriteStringFixed(buf, accChange.Name, col2width, false) - buf.WriteString(" ") - balamtColor.WriteStringFixed(buf, outBalanceString, 10, true) - buf.WriteString(" ") - runamtColor.WriteStringFixed(buf, outRunningBalanceString, 10, true) - buf.WriteString(newLine) + formatTotal := func(ct curTotal) string { + amtStr := ct.amount.StringFixedBank(2) + if ct.currency == "_" { + return amtStr + } + return ct.currency + " " + amtStr + } + + primaryTotal := formatTotal(totals[0]) + + // Colors + balamtColor := colorReset + if accChange.Balance.Sign() < 0 { + balamtColor = colorNeg + } + runamtColor := colorReset + if totals[0].amount.Sign() < 0 { + runamtColor = colorNeg + } + + // First line with primary total + buf.WriteString(trans.Date.Format(transactionDateFormat)) + buf.WriteString(" ") + colorPayee.WriteStringFixed(buf, trans.Payee, col1width, false) + buf.WriteString(" ") + colorAccount.WriteStringFixed(buf, accChange.Name, col2width, false) + buf.WriteString(" ") + balamtColor.WriteStringFixed(buf, outBalanceString, 10, true) + buf.WriteString(" ") + runamtColor.WriteStringFixed(buf, primaryTotal, 10, true) + buf.WriteString(newLine) + + // Additional lines for other currencies in running total + if len(totals) > 1 { + for _, ct := range totals[1:] { + otherTotal := formatTotal(ct) + otherColor := colorReset + if ct.amount.Sign() < 0 { + otherColor = colorNeg + } + + // Empty date/payee/account/amount columns, only total column + buf.WriteString(strings.Repeat(" ", 10)) // date + buf.WriteString(" ") + colorPayee.WriteStringFixed(buf, "", col1width, false) + buf.WriteString(" ") + colorAccount.WriteStringFixed(buf, "", col2width, false) + buf.WriteString(" ") + balamtColor.WriteStringFixed(buf, "", 10, true) + buf.WriteString(" ") + otherColor.WriteStringFixed(buf, otherTotal, 10, true) + buf.WriteString(newLine) + } } } } @@ -297,11 +382,16 @@ func PrintCSV(generalLedger []*ledger.Transaction, filterArr []string) { } if inFilter { runningBalance = runningBalance.Add(accChange.Balance) - outBalanceString := accChange.Balance.StringFixedBank() + outBalanceString := accChange.Balance.StringFixedBank(2) record := []string{trans.Date.Format(transactionDateFormat), trans.Payee, accChange.Name, - outBalanceString, + func() string { + if accChange.Currency != "" { + return accChange.Currency + " " + outBalanceString + } + return outBalanceString + }(), } if err := csvWriter.Write(record); err != nil { fmt.Fprintf(os.Stderr, "error writing record to CSV: %s", err) diff --git a/ledger/cmd/printEquity.go b/ledger/cmd/printEquity.go index 753986c2..1d527be8 100644 --- a/ledger/cmd/printEquity.go +++ b/ledger/cmd/printEquity.go @@ -8,7 +8,7 @@ import ( "time" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" "github.com/spf13/cobra" ) diff --git a/ledger/cmd/templates/template.leaderboardchart.html b/ledger/cmd/templates/template.leaderboardchart.html index 51b848e4..287f3c5d 100644 --- a/ledger/cmd/templates/template.leaderboardchart.html +++ b/ledger/cmd/templates/template.leaderboardchart.html @@ -42,7 +42,7 @@

{{.ReportName}} : {{.RangeStart.Format "2006-01-02"}} - {{.RangeEnd.Format "
- {{$acc.Balance.StringRound}} ({{$acc.Percentage}}%) + {{$acc.Balance.StringFixed(0)}} ({{$acc.Percentage}}%)
diff --git a/ledger/cmd/webHandlerAccounts.go b/ledger/cmd/webHandlerAccounts.go index 6118a89d..92fb6df8 100644 --- a/ledger/cmd/webHandlerAccounts.go +++ b/ledger/cmd/webHandlerAccounts.go @@ -75,7 +75,7 @@ func addTransactionPostHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(&tbuf, "") /* Check valid transaction is created */ - trans, perr := ledger.ParseLedger(&tbuf) + trans, perr := ledger.ParseLedger("", &tbuf) if perr != nil { http.Error(w, perr.Error(), 500) return diff --git a/ledger/cmd/webHandlerReport.go b/ledger/cmd/webHandlerReport.go index 63e64eee..23681d10 100644 --- a/ledger/cmd/webHandlerReport.go +++ b/ledger/cmd/webHandlerReport.go @@ -9,9 +9,9 @@ import ( "time" "github.com/howeyc/ledger" - "github.com/howeyc/ledger/decimal" "github.com/howeyc/ledger/ledger/cmd/internal/pdr" colorful "github.com/lucasb-eyer/go-colorful" + "github.com/shopspring/decimal" ) func getRangeAndPeriod(dateRange, dateFreq string) (start, end time.Time, period ledger.Period, err error) { @@ -82,7 +82,7 @@ func calcBalances(calcAccts []calculatedAccount, balances []*ledger.Account) (re factor := decimal.NewFromFloat(acctOp.MultiplicationFactor) fval = fval.Mul(factor) } - oval := decimal.One + oval := decimal.NewFromInt(1) if acctOp.SubAccount != "" { for _, obal := range balances { if acctOp.SubAccount == obal.Name { diff --git a/linescanner.go b/linescanner.go index 191aa5fe..f7edd850 100644 --- a/linescanner.go +++ b/linescanner.go @@ -3,14 +3,10 @@ package ledger import ( "bufio" "io" - "os" - "unsafe" ) type linescanner struct { - scanner *bufio.Scanner - unsafe bool - + scanner *bufio.Scanner filename string lineCount int } @@ -20,17 +16,6 @@ type linescanner struct { func newLineScanner(filename string, r io.Reader) *linescanner { lp := &linescanner{} lp.scanner = bufio.NewScanner(r) - if fs, fserr := os.Stat(filename); fserr == nil { - size := int(fs.Size()) - // only pre-alloc for large files - if size > bufio.MaxScanTokenSize { - // allocate large enough such that scanner in bufio doesn't need to - // move anything during scanning - size *= 2 - lp.scanner.Buffer(make([]byte, size), size) - lp.unsafe = true - } - } lp.filename = filename return lp @@ -42,15 +27,7 @@ func (lp *linescanner) Scan() bool { func (lp *linescanner) Text() string { var line string - if lp.unsafe { - if lbytes := lp.scanner.Bytes(); len(lbytes) > 0 { - line = unsafe.String(unsafe.SliceData(lbytes), len(lbytes)) - } else { - line = "" - } - } else { - line = lp.scanner.Text() - } + line = lp.scanner.Text() lp.lineCount++ return line } diff --git a/parse.go b/parse.go index 20b1ca6e..d7e81390 100644 --- a/parse.go +++ b/parse.go @@ -6,14 +6,14 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" - "sync" "time" - "unicode" - "github.com/alfredxing/calc/compute" - "github.com/howeyc/ledger/decimal" - date "github.com/joyt/godate" + "github.com/araddon/dateparse" + "github.com/expr-lang/expr" + "github.com/samber/lo" + "github.com/shopspring/decimal" ) // ParseLedgerFile parses a ledger file and returns a list of Transactions. @@ -23,61 +23,23 @@ func ParseLedgerFile(filename string) (generalLedger []*Transaction, err error) return nil, ierr } defer ifile.Close() - var mu sync.Mutex - parseLedger(filename, ifile, func(t []*Transaction, e error) (stop bool) { - if e != nil { - err = e - stop = true - return - } - - mu.Lock() - generalLedger = append(generalLedger, t...) - mu.Unlock() - return - }) - - return + return ParseLedger(filename, ifile) } // ParseLedger parses a ledger file and returns a list of Transactions. -func ParseLedger(ledgerReader io.Reader) (generalLedger []*Transaction, err error) { - parseLedger("", ledgerReader, func(t []*Transaction, e error) (stop bool) { - if e != nil { - err = e - stop = true - return - } +func ParseLedger(name string, ledgerReader io.Reader) (generalLedger []*Transaction, err error) { + blocks, err := parseBlocks(name, ledgerReader) + if err != nil { + return nil, err + } - generalLedger = append(generalLedger, t...) - return + return lo.MapErr(blocks, func(b block, _ int) (*Transaction, error) { + trans, transErr := b.parseTransaction() + if transErr != nil { + return nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", b.filename, b.lineNum, transErr) + } + return trans, nil }) - - return -} - -// ParseLedgerAsync parses a ledger file and returns a Transaction and error channels . -func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error) { - c = make(chan *Transaction) - e = make(chan error) - - go func() { - parseLedger("", ledgerReader, func(tlist []*Transaction, err error) (stop bool) { - if err != nil { - e <- err - } else { - for _, t := range tlist { - c <- t - } - } - return - }) - - e <- nil - close(c) - close(e) - }() - return c, e } type parser struct { @@ -89,37 +51,14 @@ type parser struct { strPrevDate string prevDateErr error prevDate time.Time - - transactions []Transaction - ctIdx int - postings []Account - cpIdx int -} - -const preAllocSize = 100000 -const preAllocWarn = 10 - -func (p *parser) init() { - p.transactions = make([]Transaction, preAllocSize) - p.postings = make([]Account, preAllocSize*3) - p.ctIdx = 0 - p.cpIdx = 0 -} - -func (p *parser) grow() { - if len(p.transactions)-p.ctIdx < preAllocWarn || - len(p.postings)-p.cpIdx < (preAllocWarn*3) { - p.init() - } } -func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Transaction, err error) (stop bool)) (stop bool) { +func parseBlocks(filename string, ledgerReader io.Reader) ([]block, error) { var lp parser - lp.init() lp.scanner = newLineScanner(filename, ledgerReader) - var tlist []*Transaction - + blocks := []block{} + comments := []string{} for lp.scanner.Scan() { // remove heading and tailing space from the line trimmedLine := strings.TrimSpace(lp.scanner.Text()) @@ -135,21 +74,19 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra // Skip empty lines if len(trimmedLine) == 0 { if len(currentComment) > 0 { - lp.comments = append(lp.comments, currentComment) + comments = append(comments, currentComment) } continue } before, after, split := strings.Cut(trimmedLine, " ") if !split { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), - fmt.Errorf("unable to parse payee line: %s", trimmedLine))) { - return true - } - if len(currentComment) > 0 { - lp.comments = append(lp.comments, currentComment) - } - continue + return nil, fmt.Errorf( + "%s:%d: unable to parse transaction: %w", + lp.scanner.Name(), + lp.scanner.LineNumber(), + fmt.Errorf("unable to parse payee line: %s", trimmedLine), + ) } switch before { case "account": @@ -157,38 +94,31 @@ func parseLedger(filename string, ledgerReader io.Reader, callback func(t []*Tra case "include": paths, _ := filepath.Glob(filepath.Join(filepath.Dir(lp.scanner.Name()), after)) if len(paths) < 1 { - callback(nil, fmt.Errorf("%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found"))) - return true - } - var wg sync.WaitGroup - for _, incpath := range paths { - wg.Add(1) - go func(ipath string) { - ifile, _ := os.Open(ipath) - defer ifile.Close() - if parseLedger(ipath, ifile, callback) { - stop = true - } - wg.Done() - }(incpath) + return nil, fmt.Errorf( + "%s:%d: unable to include file(%s): %w", lp.scanner.Name(), lp.scanner.LineNumber(), after, errors.New("not found")) } - wg.Wait() - if stop { - return stop + + b, err := lo.FlatMapErr(paths, func(path string, _ int) ([]block, error) { + f, _ := os.Open(path) + defer f.Close() + return parseBlocks(path, f) + }) + if err != nil { + return nil, err } + blocks = append(blocks, b...) default: - trans, transErr := lp.parseTransaction(before, after, currentComment) - if transErr != nil { - if callback(nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), transErr)) { - return true - } - continue + transDate, derr := lp.parseDate(before) + if derr != nil { + return nil, fmt.Errorf("%s:%d: unable to parse transaction: %w", lp.scanner.Name(), lp.scanner.LineNumber(), derr) } - tlist = append(tlist, trans) + + blocks = append(blocks, lp.parseBlock(transDate, after, currentComment, comments)) + comments = []string{} } } - callback(tlist, nil) - return false + + return blocks, nil } func (lp *parser) skipAccount() { @@ -206,14 +136,10 @@ func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) return lp.prevDate, lp.prevDateErr } - // try current date layout - transDate, err = time.Parse(lp.dateLayout, dateString) + // Use dateparse to handle flexible date formats + transDate, err = dateparse.ParseAny(dateString) if err != nil { - // try to find new date layout - transDate, lp.dateLayout, err = date.ParseAndGetLayout(dateString) - if err != nil { - err = fmt.Errorf("unable to parse date(%s): %w", dateString, err) - } + err = fmt.Errorf("unable to parse date(%s): %w", dateString, err) } // maybe next date is same @@ -224,90 +150,146 @@ func (lp *parser) parseDate(dateString string) (transDate time.Time, err error) return } -func (lp *parser) parseTransaction(dateString, payeeString, payeeComment string) (trans *Transaction, err error) { - transDate, derr := lp.parseDate(dateString) - if derr != nil { - return nil, derr +func (a *Account) parsePosting(trimmedLine string, comment string) (err error) { + trimmedLine = strings.TrimSpace(trimmedLine) + + // Regex groups: + // 1: account name + // 2: amount (number or parenthesized expression) + // 3: @@ converted amount + // 4: @ conversion rate + re := regexp.MustCompile( + `^(?P.+?)` + + `(?:(?:\s{2,}|\t)` + + `(?:(?P[A-Z\$]+)\s+)?` + + `(?P[\-]?\d+(?:\.\d+)?|\([0-9+\-*\/. ]+\))` + + `(?:\s*(?:@@\s*` + + `(?P[\-]?\d+(?:\.\d+)?)|@\s*` + + `(?P[\-]?\d+(?:\.\d+)?)))?)?\s*$`, + ) + + m := re.FindStringSubmatch(trimmedLine) + if m == nil { + return fmt.Errorf("invalid posting: %q", trimmedLine) } - transBal := decimal.Zero - var numEmpty int - var emptyAccIndex int - var accIndex int + a.Name = m[1] + a.Currency = m[2] + a.Comment = comment + if m[3] != "" { + program, err := expr.Compile(m[3]) + if err != nil { + return err + } + out, err := expr.Run(program, nil) + if err != nil { + return err + } + + var f float64 + switch v := out.(type) { + case int: + f = float64(v) + case int64: + f = float64(v) + case float32: + f = float64(v) + case float64: + f = v + default: + return fmt.Errorf("expression did not evaluate to a number: %T", out) + } + + a.Balance = decimal.NewFromFloat(f) + } + + // @@ explicit converted amount + if m[4] != "" { + conv, err := decimal.NewFromString(m[4]) + if err != nil { + return err + } + a.Converted = &conv + } + + // @ rate-based conversion + if m[5] != "" { + rate, err := decimal.NewFromString(m[5]) + if err != nil { + return err + } + a.ConversionFactor = &rate + } + return +} + +type block struct { + transDate time.Time + payeeString string + payeeComment string + comments []string + lines []string + filename string + lineNum int +} + +func (lp *parser) parseBlock(transDate time.Time, payeeString, payeeComment string, comments []string) block { + lines := []string{} for lp.scanner.Scan() { trimmedLine := lp.scanner.Text() + lines = append(lines, trimmedLine) + if len(trimmedLine) == 0 { + break + } + } + return block{ + transDate: transDate, + payeeString: payeeString, + payeeComment: payeeComment, + comments: comments, + lines: lines, + filename: lp.scanner.Name(), + lineNum: lp.scanner.LineNumber(), + } +} + +func (b *block) parseTransaction() (trans *Transaction, err error) { + trans = &Transaction{} + for _, trimmedLine := range b.lines { + postingComment := "" // handle comments if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >= 0 { currentComment := trimmedLine[commentIdx:] trimmedLine = trimmedLine[:commentIdx] trimmedLine = strings.TrimSpace(trimmedLine) if len(trimmedLine) == 0 { - lp.comments = append(lp.comments, currentComment) + b.comments = append(b.comments, currentComment) continue } - lp.postings[lp.cpIdx+accIndex].Comment = currentComment + postingComment = currentComment } if len(trimmedLine) == 0 { break } - if iSpace := strings.LastIndexFunc(trimmedLine, unicode.IsSpace); iSpace >= 0 { - if decbal, derr := decimal.NewFromString(trimmedLine[iSpace+1:]); derr == nil { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine[:iSpace]) - lp.postings[lp.cpIdx+accIndex].Balance = decbal - } else if iParen := strings.Index(trimmedLine, "("); iParen >= 0 { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine[:iParen]) - f, _ := compute.Evaluate(trimmedLine[iParen+1 : len(trimmedLine)-1]) - lp.postings[lp.cpIdx+accIndex].Balance = decimal.NewFromFloat(f) - } else { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine) - } - } else { - lp.postings[lp.cpIdx+accIndex].Name = strings.TrimSpace(trimmedLine) - } - - if lp.postings[lp.cpIdx+accIndex].Balance.IsZero() { - numEmpty++ - emptyAccIndex = accIndex - } - transBal = transBal.Add(lp.postings[lp.cpIdx+accIndex].Balance) - accIndex++ + posting := Account{} + posting.parsePosting(trimmedLine, postingComment) + trans.AccountChanges = append(trans.AccountChanges, posting) } - if accIndex < 2 { - err = errors.New("need at least two postings") - return + trans.Payee = b.payeeString + trans.Date = b.transDate + trans.PayeeComment = b.payeeComment + if len(b.comments) > 0 { + trans.Comments = b.comments } - if !transBal.IsZero() { - switch numEmpty { - case 0: - return nil, errors.New("unable to balance transaction: no empty account to place extra balance") - case 1: - // If there is a single empty account, then it is obvious where to - // place the remaining balance. - lp.postings[lp.cpIdx+emptyAccIndex].Balance = transBal.Neg() - default: - return nil, errors.New("unable to balance transaction: more than one account empty") - } + if err = trans.IsBalanced(); err != nil { + return nil, err } - lp.transactions[lp.ctIdx].Payee = payeeString - lp.transactions[lp.ctIdx].Date = transDate - lp.transactions[lp.ctIdx].PayeeComment = payeeComment - lp.transactions[lp.ctIdx].AccountChanges = lp.postings[lp.cpIdx : lp.cpIdx+accIndex] - lp.transactions[lp.ctIdx].Comments = lp.comments - - trans = &lp.transactions[lp.ctIdx] - - lp.comments = nil - lp.cpIdx += accIndex - lp.ctIdx++ - - lp.grow() - return } diff --git a/parseFuzz_test.go b/parseFuzz_test.go index 39d91ac5..95e6dd87 100644 --- a/parseFuzz_test.go +++ b/parseFuzz_test.go @@ -6,7 +6,7 @@ import ( "bytes" "testing" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) func FuzzParseLedger(f *testing.F) { @@ -17,11 +17,20 @@ func FuzzParseLedger(f *testing.F) { } f.Fuzz(func(t *testing.T, s string) { b := bytes.NewBufferString(s) - trans, _ := ParseLedger(b) + trans, _ := ParseLedger("", b) overall := decimal.Zero for _, t := range trans { for _, p := range t.AccountChanges { - overall = overall.Add(p.Balance) + switch { + case p.Converted != nil: + overall = overall.Add(p.Converted.Neg()) + case p.ConversionFactor != nil: + overall = overall.Add(p.Balance.Mul( + *p.ConversionFactor, + )) + default: + overall = overall.Add(p.Balance) + } } } if !overall.IsZero() { diff --git a/parse_test.go b/parse_test.go index e15bb709..eab903de 100644 --- a/parse_test.go +++ b/parse_test.go @@ -4,11 +4,10 @@ import ( "bytes" "encoding/json" "errors" - "sync" "testing" "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) type testCase struct { @@ -59,7 +58,7 @@ var testCases = []testCase{ Assets 123 `, nil, - errors.New(`:1: unable to parse transaction: unable to parse date(1970/02/31): parsing time "1970/02/31": extra text: "1970/02/31"`), + errors.New(`:1: unable to parse transaction: unable to parse date(1970/02/31): parsing time "1970/02/31": day out of range`), }, { "unbalanced error", @@ -513,12 +512,144 @@ account Assets }, nil, }, + { + "conversion factor", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK -2000.00 @ 0.5 + Assets:Wise:EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Balance: decimal.NewFromFloat(-2000.0), + ConversionFactor: p(decimal.NewFromFloat(0.5)), + }, + { + Name: "Assets:Wise:EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + }, + nil, + }, + { + "conversion", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK -2000.00 @@ 1000.00 + Assets:Wise:EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Balance: decimal.NewFromFloat(-2000.0), + Converted: p(decimal.NewFromFloat(1000)), + }, + { + Name: "Assets:Wise:EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + }, + nil, + }, + { + "conversion implicit rate", + `1970/01/01 Converted CZK to EUR + Assets:Wise:CZK CZK -2000.00 + Assets:Wise:EUR EUR 1000.00 +`, + []*Transaction{ + { + Payee: "Converted CZK to EUR", + Date: time.Unix(0, 0).UTC(), + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-2000.0), + }, + { + Name: "Assets:Wise:EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(1000.0), + Converted: p(decimal.NewFromFloat(-2000.0)), + }, + }, + }, + }, + nil, + }, + { + "conversion implicit rate USD", + `; test comment +1970/01/01 Wise Charges for: BALANCE + assets:wise EUR -8 + expenses:bank:fees EUR 8 + +; test comment +1970/01/01 Converted EUR to USD + assets:wise EUR -1000 + assets:wise USD 2060 +`, + []*Transaction{ + { + Payee: "Wise Charges for: BALANCE", + Date: time.Unix(0, 0).UTC(), + Comments: []string{"; test comment"}, + AccountChanges: []Account{ + { + Name: "assets:wise", + Currency: "EUR", + Balance: decimal.NewFromFloat(-8.0), + }, + { + Name: "expenses:bank:fees", + Currency: "EUR", + Balance: decimal.NewFromFloat(8.0), + }, + }, + }, + { + Payee: "Converted EUR to USD", + Date: time.Unix(0, 0).UTC(), + Comments: []string{"; test comment"}, + AccountChanges: []Account{ + { + Name: "assets:wise", + Currency: "EUR", + Balance: decimal.NewFromFloat(-1000.0), + }, + { + Name: "assets:wise", + Currency: "USD", + Balance: decimal.NewFromFloat(2060.0), + Converted: p(decimal.NewFromFloat(-1000)), + }, + }, + }, + }, + nil, + }, +} + +func p(d decimal.Decimal) *decimal.Decimal { + return &d } func TestParseLedger(t *testing.T) { for _, tc := range testCases { b := bytes.NewBufferString(tc.data) - transactions, err := ParseLedger(b) + transactions, err := ParseLedger("", b) if (err != nil && tc.err == nil) || (err != nil && tc.err != nil && err.Error() != tc.err.Error()) { t.Errorf("Error: expected `%s`, got `%s`", tc.err, err) } @@ -530,54 +661,134 @@ func TestParseLedger(t *testing.T) { } } -func TestParseLedgerAsync(t *testing.T) { - buf := bytes.NewBufferString(`; test -account bam:bam - subacc line ; sub comment - another subacc line - -1970/01/01 Payee - Assets 50 - Expenses - -1970/02/30 Error ; oops - Assets 30 - Expenses - -1970/01/01bbafafdaf;bad comment - Assets 20 - Expenses - -account endofledger`) - - tc, ec := ParseLedgerAsync(buf) - - var trans []*Transaction - var errors []error - - var wg sync.WaitGroup - wg.Add(2) - go func() { - for t := range tc { - trans = append(trans, t) - } - wg.Done() - }() - go func() { - for e := range ec { - errors = append(errors, e) - } - wg.Done() - }() - wg.Wait() - - if len(trans) < 1 || len(errors) < 1 { - t.Error("async parse failed") - } -} - func BenchmarkParseLedger(b *testing.B) { for b.Loop() { _, _ = ParseLedgerFile("testdata/ledgerBench.dat") } } + +func TestAccount_parsePosting(t *testing.T) { + tests := []struct { + name string + trimmedLine string + want Account + wantErr bool + }{ + { + "simple", + "Expense 123", + Account{Name: "Expense", Balance: decimal.NewFromFloat(123.0)}, + false, + }, + { + "empty", + "Expense", + Account{Name: "Expense", Balance: decimal.NewFromFloat(0.0)}, + false, + }, + { + "spaces", + "Expense:Cranks Unlimited 10", + Account{Name: "Expense:Cranks Unlimited", Balance: decimal.NewFromFloat(10.0)}, + false, + }, + { + "multiply", + "Expense (123*2)", + Account{Name: "Expense", Balance: decimal.NewFromFloat(246.0)}, + false, + }, + { + "slash", + "Expense/test 158", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(158.0)}, + false, + }, + { + "negative", + "Expense/test -158", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(-158.0)}, + false, + }, + { + "math", + "Expense:Bank of:Money (123*2+3)", + Account{Name: "Expense:Bank of:Money", Balance: decimal.NewFromFloat(249.0)}, + false, + }, + { + "math with spaces", + "Expense/test (123 * 3)", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(123 * 3)}, + false, + }, + { + "converted", + "Expense/test 158 @@ 200", + Account{Name: "Expense/test", Balance: decimal.NewFromFloat(158.0), Converted: p(decimal.NewFromFloat(200.0))}, + false, + }, + { + "conversion", + "Expense/test 100 @ 2", + Account{Name: "Expense/test", Currency: "", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, + false, + }, + { + "conversion heirarchy", + "Assets:Wise:CZK -2000.00 @ 0.5", + Account{Name: "Assets:Wise:CZK", Balance: decimal.NewFromFloat(-2000.0), ConversionFactor: p(decimal.NewFromFloat(0.5))}, + false, + }, + { + "negative", + "Expense/test EUR -158", + Account{Name: "Expense/test", Currency: "EUR", Balance: decimal.NewFromFloat(-158.0)}, + false, + }, + { + "math", + "Expense:Bank of:Money USD (123*2+3)", + Account{Name: "Expense:Bank of:Money", Currency: "USD", Balance: decimal.NewFromFloat(249.0)}, + false, + }, + { + "math with spaces", + "Expense/test CZK (123 * 3)", + Account{Name: "Expense/test", Currency: "CZK", Balance: decimal.NewFromFloat(123 * 3)}, + false, + }, + { + "converted", + "Expense/test USD 158 @@ 200", + Account{Name: "Expense/test", Currency: "USD", Balance: decimal.NewFromFloat(158.0), Converted: p(decimal.NewFromFloat(200.0))}, + false, + }, + { + "conversion", + "Expense/test $ 100 @ 2", + Account{Name: "Expense/test", Currency: "$", Balance: decimal.NewFromFloat(100.0), ConversionFactor: p(decimal.NewFromFloat(2.0))}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := Account{} + gotErr := a.parsePosting(tt.trimmedLine, "") + if gotErr != nil { + if !tt.wantErr { + t.Errorf("parsePosting() failed: %v", gotErr) + } + return + } + aJson, _ := json.Marshal(a) + wantJson, _ := json.Marshal(tt.want) + if string(aJson) != string(wantJson) { + t.Errorf("got %+v wanted %+v", string(aJson), string(wantJson)) + } + if tt.wantErr { + t.Fatal("parsePosting() succeeded unexpectedly") + } + }) + } +} diff --git a/transaction.go b/transaction.go new file mode 100644 index 00000000..717f3c7b --- /dev/null +++ b/transaction.go @@ -0,0 +1,189 @@ +package ledger + +import ( + "errors" + + "github.com/shopspring/decimal" +) + +var ( + ErrNeedAtLeastTwoPostings = errors.New("need at least two postings") + ErrNoEmptyAccountForExtraBalance = errors.New("unable to balance transaction: no empty account to place extra balance") + ErrMoreThanOneEmptyAccountInTx = errors.New("unable to balance transaction: more than one account empty") +) + +func (t *Transaction) IsBalanced() error { + if len(t.AccountChanges) < 2 { + return ErrNeedAtLeastTwoPostings + } + + if err := t.inferConversionFactorForTwoCurrencyTx(); err != nil { + return err + } + + transBal := decimal.Zero + var numEmpty int + var emptyAccIndex int + + for i, acc := range t.AccountChanges { + if acc.Balance.IsZero() { + numEmpty++ + emptyAccIndex = i + } + + switch { + case acc.Converted != nil: + transBal = transBal.Add(acc.Converted.Neg()) + case acc.ConversionFactor != nil: + transBal = transBal.Add(acc.Balance.Mul(*acc.ConversionFactor)) + default: + transBal = transBal.Add(acc.Balance) + } + } + + if !transBal.IsZero() { + switch numEmpty { + case 0: + return ErrNoEmptyAccountForExtraBalance + case 1: + // If there is a single empty account, then it is obvious where to + // place the remaining balance. + t.AccountChanges[emptyAccIndex].Balance = transBal.Neg() + default: + return ErrMoreThanOneEmptyAccountInTx + } + } + + return nil +} + +func (t *Transaction) inferConversionFactorForTwoCurrencyTx() error { + type currencyGroup struct { + indices []int + } + + currencyMap := make(map[string]*currencyGroup) + + getCurrencyKey := func(a *Account) string { + if a.Converted != nil { + // TODO: explicit currency for conversion? + return a.Currency + } + return a.Currency + } + + for i := range t.AccountChanges { + acc := &t.AccountChanges[i] + key := getCurrencyKey(acc) + if key == "" { + return nil + } + group, ok := currencyMap[key] + if !ok { + group = ¤cyGroup{} + currencyMap[key] = group + } + group.indices = append(group.indices, i) + } + + if len(currencyMap) != 2 { + return nil + } + + var ( + curKeys [2]string + groups [2]*currencyGroup + ) + + // Collect keys to deterministically choose base/other currency + i := 0 + for k, g := range currencyMap { + if i >= 2 { + break + } + curKeys[i] = k + groups[i] = g + i++ + } + + // Assign base currency as the one with the lower sort order + if curKeys[1] < curKeys[0] { + curKeys[0], curKeys[1] = curKeys[1], curKeys[0] + groups[0], groups[1] = groups[1], groups[0] + } + + var baseCurIdx, otherCurIdx int + hasConv0 := false + for _, idx := range groups[0].indices { + if t.AccountChanges[idx].ConversionFactor != nil { + hasConv0 = true + break + } + } + hasConv1 := false + for _, idx := range groups[1].indices { + if t.AccountChanges[idx].ConversionFactor != nil { + hasConv1 = true + break + } + } + + switch { + case hasConv0 && hasConv1: + return nil + case hasConv0: + baseCurIdx, otherCurIdx = 1, 0 + case hasConv1: + baseCurIdx, otherCurIdx = 0, 1 + default: + baseCurIdx, otherCurIdx = 0, 1 + } + + sumForGroup := func(g *currencyGroup) (decimal.Decimal, error) { + total := decimal.Zero + for _, idx := range g.indices { + acc := &t.AccountChanges[idx] + if acc.Converted != nil { + total = total.Add(acc.Converted.Neg()) + } else if acc.ConversionFactor != nil { + total = total.Add(acc.Balance.Mul(*acc.ConversionFactor)) + } else { + total = total.Add(acc.Balance) + } + } + return total, nil + } + + sumBase, _ := sumForGroup(groups[baseCurIdx]) + sumOtherRaw := decimal.Zero + for _, idx := range groups[otherCurIdx].indices { + acc := &t.AccountChanges[idx] + if acc.Converted != nil || acc.ConversionFactor != nil { + switch { + case acc.Converted != nil: + sumOtherRaw = sumOtherRaw.Add(acc.Converted.Neg()) + case acc.ConversionFactor != nil: + sumOtherRaw = sumOtherRaw.Add(acc.Balance.Mul(*acc.ConversionFactor)) + } + } else { + sumOtherRaw = sumOtherRaw.Add(acc.Balance) + } + } + + if sumOtherRaw.IsZero() { + return nil + } + if sumBase.Add(sumOtherRaw).IsZero() { + return nil + } + + for _, idx := range groups[otherCurIdx].indices { + acc := &t.AccountChanges[idx] + if acc.ConversionFactor == nil && acc.Converted == nil { + conv := acc.Balance.Mul(sumBase).Div(sumOtherRaw) + acc.Converted = &conv + } + } + + return nil +} diff --git a/transaction_test.go b/transaction_test.go new file mode 100644 index 00000000..2a630312 --- /dev/null +++ b/transaction_test.go @@ -0,0 +1,207 @@ +package ledger + +import ( + "testing" + + "github.com/shopspring/decimal" +) + +func TestIsBalanced(t *testing.T) { + tests := []struct { + name string + tx *Transaction + wantErr error + wantBalances []decimal.Decimal + }{ + { + name: "errors on too few postings", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + }, + }, + wantErr: ErrNeedAtLeastTwoPostings, + wantBalances: nil, + }, + { + name: "no empty account error", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + { + Name: "Expenses:Food", + Balance: decimal.NewFromInt(-5), + }, + }, + }, + wantErr: ErrNoEmptyAccountForExtraBalance, + wantBalances: nil, + }, + { + name: "more than one empty account error", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(10), + }, + { + Name: "Expenses:Food", + }, + { + Name: "Equity:OpeningBalances", + }, + }, + }, + wantErr: ErrMoreThanOneEmptyAccountInTx, + wantBalances: nil, + }, + { + name: "single empty account gets balancing amount", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Expenses:Food", + }, + }, + }, + wantErr: nil, + wantBalances: []decimal.Decimal{decimal.NewFromInt(-10), decimal.NewFromInt(10)}, + }, + { + name: "already balanced with no empty account", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Expenses:Food", + Balance: decimal.NewFromInt(10), + }, + }, + }, + wantErr: nil, + wantBalances: []decimal.Decimal{decimal.NewFromInt(-10), decimal.NewFromInt(10)}, + }, + { + name: "two currency implicit conversion factor inferred", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + { + name: "two currency implicit conversion factor inferred multiple", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + { + Name: "Assets:otherBank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(3), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + { + name: "does not infer conversion factor for three currencies", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Bank:USD", + Currency: "USD", + Balance: decimal.NewFromInt(-10), + }, + { + Name: "Assets:Bank:EUR", + Currency: "EUR", + Balance: decimal.NewFromInt(5), + }, + { + Name: "Assets:Bank:GBP", + Currency: "GBP", + Balance: decimal.NewFromInt(3), + }, + }, + }, + wantErr: ErrNoEmptyAccountForExtraBalance, + wantBalances: nil, + }, + { + name: "decimal precision bug", + tx: &Transaction{ + AccountChanges: []Account{ + { + Name: "Assets:Wise:CZK", + Currency: "CZK", + Balance: decimal.NewFromFloat(-2003.0), + }, + { + Name: "Assets:Wise:EUR", + Currency: "EUR", + Balance: decimal.NewFromFloat(1000.0), + }, + }, + }, + wantErr: nil, + wantBalances: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := tt.tx.IsBalanced() + if err != tt.wantErr { + t.Fatalf("expected error %v, got %v", tt.wantErr, err) + } + + if tt.wantBalances != nil { + if len(tt.tx.AccountChanges) != len(tt.wantBalances) { + t.Fatalf("expected %d account balances, got %d", len(tt.wantBalances), len(tt.tx.AccountChanges)) + } + for i, want := range tt.wantBalances { + if !tt.tx.AccountChanges[i].Balance.Equal(want) { + t.Fatalf("account %d: expected balance %s, got %s", i, want.String(), tt.tx.AccountChanges[i].Balance.String()) + } + } + } + }) + } +} diff --git a/types.go b/types.go index ccfc0efa..f141afca 100644 --- a/types.go +++ b/types.go @@ -3,14 +3,21 @@ package ledger import ( "time" - "github.com/howeyc/ledger/decimal" + "github.com/shopspring/decimal" ) // Account holds the name and balance type Account struct { - Name string - Balance decimal.Decimal - Comment string + Name string + // Default "" for no currency/token displayed + Currency string + Balance decimal.Decimal + Comment string + + // Balance converted using @@ notation + Converted *decimal.Decimal + // Conversion factor using @ notation + ConversionFactor *decimal.Decimal } // Transaction is the basis of a ledger. The ledger holds a list of transactions.