server/controllers/all.ts
Rama main · réplica de lectura del repositorio original.
import crypto from 'crypto';
import type express from 'express';
import {
DEFAULT_ACCOUNT_ID,
DEMO_MODE,
DUPLICATE_LAX_MODE,
DUPLICATE_THRESHOLD,
} from '../../shared/settings';
import {
assert,
asyncErr,
getErrorCode,
isAppriseApiEnabled,
isEmailEnabled,
KError,
makeLogger,
UNKNOWN_TRANSACTION_TYPE,
unwrap,
} from '../helpers';
import diffAccount from '../lib/diff-accounts';
import diffTransactions from '../lib/diff-transactions';
import { ConfigGhostSettings, getAll as getAllInstanceProperties } from '../lib/instance';
import {
Access,
Account,
Alert,
AppliedRecurringTransaction,
Budget,
Category,
DuplicatesIgnored,
type MinimalTransaction,
RecurringTransaction,
Setting,
Transaction,
TransactionRule,
User,
View,
} from '../models';
import runDataMigrations from '../models/data-migrations';
import { bankVendorByUuid, getBankVendors } from '../providers';
import DefaultSettings from '../shared/default-settings';
import { validatePassword } from '../shared/helpers';
import type { DuplicatesPairs } from '../shared/types';
import { type AllData, type ClientAccess, cleanData, type Remapping } from './helpers';
import { isDemoEnabled } from './instance';
import { ofxToKresus } from './ofx';
import type { IdentifiedRequest } from './routes';
const log = makeLogger('controllers/all');
const ERR_MSG_LOADING_ALL = 'Error when loading all Kresus data';
interface GetAllDataOptions {
isExport?: boolean;
cleanPassword?: boolean;
}
type StartupTask = () => Promise<void>;
type UserId = number;
// Startup tasks are cleanup tasks that will be run the next time a user will run the /all initial
// request.
const STARTUP_TASKS: Record<UserId, StartupTask[]> = {};
// Registers a new startup task for the given user.
export function registerStartupTask(userId: UserId, f: StartupTask) {
STARTUP_TASKS[userId] = STARTUP_TASKS[userId] || [];
STARTUP_TASKS[userId].push(f);
}
// Run startup tasks for the user, if any.
async function runStartupTasks(userId: number) {
if (STARTUP_TASKS[userId]) {
while (STARTUP_TASKS[userId].length) {
const task = unwrap(STARTUP_TASKS[userId].pop());
await task();
}
}
}
async function getAllData(userId: number, options: GetAllDataOptions = {}): Promise<AllData> {
const { isExport = false, cleanPassword = true } = options;
let ret: AllData = {
accounts: [],
accesses: [],
alerts: [],
categories: [],
transactions: [],
settings: [],
instance: {},
recurringTransactions: [],
views: [],
bankVendors: [],
};
const accesses = await Access.all(userId);
for (const access of accesses) {
const clientAccess: ClientAccess = { ...access };
// Just keep the name and the value of the field.
clientAccess.fields = (access.fields || []).map(({ name, value }) => {
return { name, value };
});
if (cleanPassword) {
clientAccess.fields = clientAccess.fields.filter(f => f.name !== 'password');
delete clientAccess.session;
}
if (!isExport) {
// Process enabled status only for the /all request.
clientAccess.enabled = access.isEnabled();
delete clientAccess.session;
}
const bank = bankVendorByUuid(clientAccess.vendorId);
if (bank && bank.name) {
clientAccess.label = bank.name;
}
ret.accesses.push(clientAccess);
}
// On export we don't want the balance to be returned for manual accounts, otherwise
// on import the balance won't be automatically re-computed on transactions
// creation/update/deletion.
ret.accounts = await Account.all(userId, !isExport);
ret.categories = await Category.all(userId);
ret.transactions = await Transaction.all(userId);
ret.settings = await Setting.all(userId);
ret.recurringTransactions = await RecurringTransaction.all(userId);
ret.views = await View.all(userId);
if (isExport) {
ret.budgets = await Budget.all(userId);
// This fetches the associated conditions and actions data, so no need
// to join explicitly here.
ret.transactionRules = await TransactionRule.allOrdered(userId);
// We only need to export the applied recurring transactions from the current month since
// the recurring transactions won't be created for the past at next poll.
const now = new Date();
ret.appliedRecurringTransactions = await AppliedRecurringTransaction.byMonthAndYear(
userId,
now.getMonth(),
now.getFullYear()
);
const ignoredDuplicatesPairs = (await DuplicatesIgnored.all(userId)).map(pair => [
pair.transactionId,
pair.otherTransactionId,
]) satisfies DuplicatesPairs;
ret.duplicates = {
ignored: ignoredDuplicatesPairs,
};
} else {
ret.bankVendors = getBankVendors();
ret.instance = await getAllInstanceProperties();
// Note: the duplicates (detected and ignored ones) are lazy-loaded through the
// /duplicates endpoint and therefore not part of this payload.
const user = await User.find(userId);
if (user) {
ret.user = user;
}
}
if (isExport || isEmailEnabled() || isAppriseApiEnabled()) {
ret.alerts = await Alert.all(userId);
} else {
ret.alerts = [];
}
if (isExport) {
ret = cleanData(ret);
}
return ret;
}
export async function all(req: IdentifiedRequest<any>, res: express.Response) {
try {
const { id: userId } = req.user;
await runStartupTasks(userId);
const ret = await getAllData(userId);
res.status(200).json(ret);
} catch (err) {
err.code = ERR_MSG_LOADING_ALL;
asyncErr(res, err, 'when loading all data');
}
}
const ENCRYPTION_ALGORITHM = 'aes-256-ctr';
const ENCRYPTED_CONTENT_TAG = Buffer.from('KRE');
function encryptData(data: Record<string, unknown>, passphrase: string) {
assert(process.kresus.salt !== null, 'must have provided a salt');
const initVector = crypto.randomBytes(16);
const key = crypto.pbkdf2Sync(passphrase, process.kresus.salt, 100000, 32, 'sha512');
const cipher = crypto.createCipheriv(ENCRYPTION_ALGORITHM, key, initVector);
return Buffer.concat([
initVector,
ENCRYPTED_CONTENT_TAG,
cipher.update(JSON.stringify(data)),
cipher.final(),
]).toString('base64');
}
function decryptData(data: string, passphrase: string) {
assert(process.kresus.salt !== null, 'must have provided a salt');
const rawData = Buffer.from(data, 'base64');
const [initVector, tag, encrypted] = [
rawData.slice(0, 16),
rawData.slice(16, 16 + 3),
rawData.slice(16 + 3),
];
if (tag.toString() !== ENCRYPTED_CONTENT_TAG.toString()) {
throw new KError(
'submitted file is not a valid kresus encrypted file',
400,
getErrorCode('INVALID_ENCRYPTED_EXPORT')
);
}
const key = crypto.pbkdf2Sync(passphrase, process.kresus.salt, 100000, 32, 'sha512');
const decipher = crypto.createDecipheriv(ENCRYPTION_ALGORITHM, key, initVector);
return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString();
}
export async function export_(req: IdentifiedRequest<any>, res: express.Response) {
try {
const { id: userId } = req.user;
let passphrase: string | null = null;
if (req.body.encrypted) {
if (typeof req.body.passphrase !== 'string' || !req.body.passphrase) {
throw new KError('missing parameter "passphrase"', 400);
}
if (process.kresus.salt === null) {
throw new KError(
"server hasn't been configured for encryption; " +
'please ask your administrator to provide a salt'
);
}
passphrase = req.body.passphrase;
assert(passphrase !== null, 'passphrase must be set here');
// Check password strength
if (!validatePassword(passphrase)) {
throw new KError('submitted passphrase is too weak', 400);
}
}
const data = await getAllData(userId, {
isExport: true,
cleanPassword: !passphrase,
});
let ret = {};
if (passphrase) {
const encryptedData = encryptData(data, passphrase);
ret = {
encrypted: true,
data: encryptedData,
};
} else {
ret = {
encrypted: false,
data,
};
}
res.status(200).json(ret);
} catch (err) {
err.code = ERR_MSG_LOADING_ALL;
asyncErr(res, err, 'when exporting data');
}
}
type AnyObject = { [key: string]: AnyObject } | any;
function applyRenamings(model: any): (arg: AnyObject) => AnyObject {
if (typeof model.renamings === 'undefined') {
return obj => obj;
}
return (obj: AnyObject): AnyObject => {
for (const from of Object.keys(model.renamings)) {
const to = model.renamings[from];
if (typeof obj[from] !== 'undefined') {
if (typeof obj[to] === 'undefined') {
obj[to] = obj[from];
}
delete obj[from];
}
}
return obj;
};
}
export function parseDate(date: any) {
switch (typeof date) {
case 'string': {
const parsedDate = Date.parse(date);
if (!isNaN(parsedDate)) {
return new Date(parsedDate);
}
break;
}
case 'number':
if (!isNaN(date) && date > -8640000000000000 && date < 8640000000000000) {
return new Date(date);
}
break;
default:
if (date instanceof Date) {
return date;
}
}
return null;
}
type AccessRemapping = {
[key: number]: {
// Remapped ID of the Kresus access in our database.
id: number;
// Was the access known, i.e. it exists in Kresus' database?
wasKnown: boolean;
// Optional vendor id of the access.
vendorId?: string;
};
};
// Import the given data from the `world` object, created by interpreting the JSON created from a
// call to `export_()` in this same file.
//
// If `dontCreateAccess` is true, it is assumed that the access id refers to a valid access that is
// known in the database. Otherwise, then the access is created.
//
// Note: destroy the input `world` argument by changing its fields in place.
export async function importData(userId: number, world: any, dontCreateAccess?: boolean) {
world.accesses = (world.accesses || []).map(applyRenamings(Access));
world.accounts = (world.accounts || []).map(applyRenamings(Account));
world.alerts = (world.alerts || []).map(applyRenamings(Alert));
world.budgets = (world.budgets || []).map(applyRenamings(Budget));
world.categories = (world.categories || []).map(applyRenamings(Category));
// Keep backward compat with 'operations'.
world.transactions = (world.transactions || world.operations || []).map(
applyRenamings(Transaction)
);
world.settings = (world.settings || []).map(applyRenamings(Setting));
world.transactionRules = world.transactionRules || [];
world.recurringTransactions = world.recurringTransactions || [];
world.appliedRecurringTransactions = world.appliedRecurringTransactions || [];
world.views = world.views || [];
// Static data.
// Keep backward compat with 'operationtypes'.
world.transactiontypes = world.transactiontypes || world.operationtypes || [];
// Importing only known settings prevents assertion errors in the client when
// importing Kresus data in an older version of kresus.
world.settings = world.settings.filter((s: any) => DefaultSettings.has(s.key)) || [];
// Exports made before the ignored duplicates were exported don't have a `duplicates` object.
world.ignoredDuplicates = world.duplicates?.ignored || [];
log.info(`Importing:
accesses: ${world.accesses.length}
accounts: ${world.accounts.length}
alerts: ${world.alerts.length}
budgets: ${world.budgets.length}
categories: ${world.categories.length}
transaction-types: ${world.transactiontypes.length}
settings: ${world.settings.length}
transactions: ${world.transactions.length}
rules: ${world.transactionRules.length}
recurring-transactions: ${world.recurringTransactions.length}
applied-recurring-transactions: ${world.appliedRecurringTransactions.length}
views: ${world.views.length},
ignoredDuplicates ${world.ignoredDuplicates.length}
`);
log.info('Import accesses...');
const accessMap: AccessRemapping = {};
for (const access of world.accesses) {
const accessId = access.id;
delete access.id;
delete access.userId;
// Support legacy "customFields" value.
if (typeof access.customFields === 'string' && !access.fields) {
try {
access.fields = JSON.parse(access.customFields);
} catch (e) {
log.error('Invalid JSON customFields, ignoring fields:', e.toString());
}
}
if (!(access.fields instanceof Array)) {
access.fields = [];
}
// Support legacy 'login' and 'password' values.
if (access.login) {
if (
!access.fields.some(
(f: unknown) => typeof f === 'object' && f && 'name' in f && f.name === 'login'
)
) {
access.fields.push({ name: 'login', value: access.login });
}
delete access.login;
}
if (access.password) {
if (
!access.fields.some(
(f: unknown) =>
typeof f === 'object' && f && 'name' in f && f.name === 'password'
)
) {
access.fields.push({ name: 'password', value: access.password });
}
delete access.password;
}
const sanitizedCustomFields: { name: string; value: string }[] = [];
for (const { name, value } of access.fields) {
if (typeof name !== 'string') {
log.warn('Ignoring access field because of non-string "name" property.');
continue;
}
if (typeof value !== 'string') {
log.warn(
`Ignoring access field for key ${name} because of non-string "value" property`
);
continue;
}
sanitizedCustomFields.push({ name, value });
}
access.fields = sanitizedCustomFields;
const accessLogin = sanitizedCustomFields.find(f => f.name === 'login');
// Try to match the vendor id and login against an existing pair in the database.
const foundKnown: Access | null = accessLogin
? await Access.byCredentials(userId, {
uuid: access.vendorId,
login: accessLogin.value,
})
: null;
if (foundKnown !== null) {
// The access was already known: don't reimport it.
accessMap[accessId] = {
id: foundKnown.id,
wasKnown: true,
vendorId: foundKnown.vendorId,
};
log.info(
`Ignoring import of access ${access.vendorId} (${
accessLogin ? accessLogin.value : 'no login'
}) as it already exists.`
);
continue;
}
if (dontCreateAccess) {
// The access id given from the `world` object is valid, according to this function's
// contract, so the mapping doesn't need to redirect to another access we created.
accessMap[accessId] = { id: accessId, wasKnown: true };
log.info(
`Not creating new known access ${access.vendorId} (${
accessLogin ? accessLogin.value : 'no login'
}) because it's explicitly marked as known.`
);
} else {
const created = await Access.create(userId, access);
accessMap[accessId] = { id: created.id, wasKnown: false };
log.info(
`Creating new unknown access ${access.vendorId} (${
accessLogin ? accessLogin.value : 'no login'
}).`
);
}
}
log.info('Done.');
log.info('Import accounts...');
const accountIdToAccount: Map<number, number> = new Map();
const vendorToOwnAccountId: Map<string, number> = new Map();
for (const account of world.accounts) {
if (typeof accessMap[account.accessId] === 'undefined') {
log.warn('Ignoring orphan account:\n', account);
continue;
}
const accountId = account.id;
// Create a copy of the account. We should use structuredClone but for some reasons our
// tests are passing models instead of literal objects.
const accountCopy = JSON.parse(JSON.stringify(account));
delete accountCopy.id;
delete accountCopy.userId;
// For an initial import which does not come from Kresus (ex: a
// handmade JSON file), there might be no lastCheckDate.
accountCopy.lastCheckDate = parseDate(accountCopy.lastCheckDate);
if (accountCopy.lastCheckDate === null) {
let latestTransactionDate: Date | null = null;
if (world.transactions) {
const accountTransactions = world.transactions.filter(
(tr: any) => tr.accountId === accountId
);
for (const tr of accountTransactions) {
const trDate = parseDate(tr.date);
if (
trDate !== null &&
(latestTransactionDate === null || trDate > latestTransactionDate)
) {
latestTransactionDate = trDate;
}
}
}
accountCopy.lastCheckDate = latestTransactionDate || new Date();
}
// If the access containing this account was known, then try to match this account against
// one known in the database.
const accessRemap = accessMap[accountCopy.accessId];
accountCopy.accessId = accessRemap?.id;
if (accessRemap?.wasKnown) {
const knownAccounts = await Account.byAccess(userId, {
id: accessRemap.id,
});
if (typeof knownAccounts !== 'undefined') {
const vendorId = accessRemap.vendorId ?? undefined;
const diffResult = diffAccount(knownAccounts, [accountCopy], {
vendorId,
});
// There can be at most one perfect match, since we provided only one account.
if (diffResult.perfectMatches.length === 1) {
const firstMatchPair = diffResult.perfectMatches[0];
const knownAccount = firstMatchPair[0];
// The account was found as a perfect match: do not import it, and reuse it
// instead.
accountIdToAccount.set(accountId, knownAccount.id);
vendorToOwnAccountId.set(accountCopy.vendorAccountId, knownAccount.id);
log.info(
`Ignoring import of account ${accountCopy.label} (${accountCopy.vendorAccountId}) as it already exists.`
);
continue;
}
}
}
log.info(
`Importing new unknown account ${accountCopy.label} (${accountCopy.vendorAccountId}).`
);
const created = await Account.create(userId, accountCopy);
accountIdToAccount.set(accountId, created.id);
vendorToOwnAccountId.set(created.vendorAccountId, created.id);
}
log.info('Done.');
log.info('Import views...');
const viewsMap: Remapping = {};
const existingViews = await View.all(userId);
for (const view of world.views) {
const viewId = view.id;
delete view.id;
if (!view.createdByUser) {
// Views created by the user are views created automatically by Kresus to group accounts.
// Since new automatic views were created automatically when importing accounts, we
// should not recreate them here.
// However, we still need to map the view ids (for budgets at least).
// Try to retrieve the corresponding view created automatically.
const viewAccountsRemapped = view.accounts.map((vAcc: any) =>
accountIdToAccount.get(vAcc.accountId)
);
const correspondingView = existingViews.find(ev => {
if (ev.accounts.length === view.accounts.length) {
const evAccounts = ev.accounts.map(a => a.accountId);
for (const accId of viewAccountsRemapped) {
if (!evAccounts.includes(accId)) {
return false;
}
}
return true;
}
return false;
});
if (correspondingView) {
viewsMap[viewId] = correspondingView.id;
log.warn(
'Not re-creating view not created by user as there is already one corresponding view:\n',
view
);
continue;
}
}
const viewAccounts = [];
for (const viewAcc of view.accounts) {
if (
typeof viewAcc.accountId !== 'undefined' &&
accountIdToAccount.has(viewAcc.accountId)
) {
viewAcc.accountId = accountIdToAccount.get(viewAcc.accountId);
viewAccounts.push(viewAcc);
} else {
log.warn('Ignoring account in view:\n', view);
}
}
if (viewAccounts.length === 0) {
log.warn('Ignoring view without accounts:\n', view);
continue;
}
const created = await View.create(userId, Object.assign(view, { accounts: viewAccounts }));
viewsMap[viewId] = created.id;
}
log.info('Done.');
log.info('Import categories...');
const existingCategories = await Category.all(userId);
const existingCategoriesMap = new Map();
for (const category of existingCategories) {
existingCategoriesMap.set(category.label, category);
}
const categoryMap: Remapping = {};
for (const category of world.categories) {
const catId = category.id;
delete category.id;
delete category.userId;
if (existingCategoriesMap.has(category.label)) {
const existing = existingCategoriesMap.get(category.label);
categoryMap[catId] = existing.id;
} else {
const created = await Category.create(userId, category);
categoryMap[catId] = created.id;
}
}
log.info('Done.');
log.info('Import budgets...');
const makeBudgetKey = (b: Budget) => `${b.viewId}-${b.categoryId}-${b.year}-${b.month}`;
const existingBudgets = await Budget.all(userId);
const existingBudgetsMap = new Map();
for (const budget of existingBudgets) {
existingBudgetsMap.set(makeBudgetKey(budget), budget);
}
let defaultViewId = -1;
let lookForDefaultView = true;
for (const importedBudget of world.budgets) {
if (typeof importedBudget.viewId !== 'number') {
if (defaultViewId === -1 && lookForDefaultView) {
// For budgets pre-existing views, there is no viewId set. We need to retrieve
// it from the defaultAccountId (the one imported) or the first account imported.
const defaultAccountImportedSetting = world.settings.find(
(setting: any) => setting && setting.key === DEFAULT_ACCOUNT_ID
);
let defaultImportedAccountId: number | undefined;
if (defaultAccountImportedSetting) {
defaultImportedAccountId = accountIdToAccount.get(
defaultAccountImportedSetting.value
);
} else {
// Find the first checking account otherwise the first account.
const bestGuessAccountFromImport =
world.accounts.find((acc: any) => acc.type === 'account-type.checking') ||
world.accounts[0];
defaultImportedAccountId = accountIdToAccount.get(
bestGuessAccountFromImport.id
);
}
if (typeof defaultImportedAccountId === 'number') {
const allViews = await View.all(userId);
const defaultAccountAssociatedView = allViews.find(view => {
return (
view.accounts.length === 1 &&
view.accounts[0].accountId === defaultImportedAccountId
);
});
if (defaultAccountAssociatedView) {
defaultViewId = defaultAccountAssociatedView.id;
} else {
lookForDefaultView = false;
}
} else {
lookForDefaultView = false;
}
}
if (defaultViewId === -1) {
// We don't know what to do with this budget without any account/view
// to associate it with.
log.warn('No view to associate a budget with, skipping budget import…');
continue;
}
// Set the default view id as the budget's viewId.
importedBudget.viewId = defaultViewId;
} else {
if (!viewsMap[importedBudget.viewId]) {
log.warn('No view to associate a budget with, skipping budget import…');
continue;
}
importedBudget.viewId = viewsMap[importedBudget.viewId];
}
// Note the order here: first map to the actual category id, so the
// map lookup thereafter uses an existing category id.
importedBudget.categoryId = categoryMap[importedBudget.categoryId];
const existingBudget = existingBudgetsMap.get(makeBudgetKey(importedBudget));
if (existingBudget) {
if (
!existingBudget.threshold ||
existingBudget.threshold !== importedBudget.threshold
) {
await Budget.update(userId, existingBudget.id, {
threshold: importedBudget.threshold,
});
}
} else {
delete importedBudget.id;
delete importedBudget.userId;
const newBudget = await Budget.create(userId, importedBudget);
// There could be duplicates in the import (see #1051), ensure we don't try
// to import the same budget twice.
existingBudgetsMap.set(makeBudgetKey(newBudget), newBudget);
}
}
log.info('Done.');
// No need to import transaction types.
// importedTypesMap is used to set type to imported transactions (backward compatibility).
const importedTypes = world.transactiontypes || [];
const importedTypesMap = new Map();
for (const type of importedTypes) {
importedTypesMap.set(type.id.toString(), type.name);
}
log.info('Import transactions...');
let perfectMatchMaxDateThreshold = 0;
// If the user chose to loosen the duplicate detection algo, use the duplicate threshold for the perfect match algo instead of default 0 value
const enableDuplicateLaxModeSetting = world.settings.find(
(s: Setting) => s.key === DUPLICATE_LAX_MODE
);
const enableDuplicateLaxMode =
(enableDuplicateLaxModeSetting
? enableDuplicateLaxModeSetting.value
: unwrap(DefaultSettings.get(DUPLICATE_LAX_MODE))) === 'true';
if (enableDuplicateLaxMode) {
const duplicateThresholdSetting = world.settings.find(
(s: Setting) => s.key === DUPLICATE_THRESHOLD
);
// The duplicate threshold is stored in hours, convert it to days.
const duplicateThreshold = duplicateThresholdSetting
? duplicateThresholdSetting.value
: unwrap(DefaultSettings.get(DUPLICATE_THRESHOLD));
perfectMatchMaxDateThreshold = Math.round(Number.parseInt(duplicateThreshold, 10) / 24);
}
const newByAccountId: Map<number, MinimalTransaction[]> = new Map();
// Maps the transaction ids as they were in the imported data to the ids they were given when
// inserted in database. Since the transactions are inserted per account, and only some of them
// are actually inserted, the previous id is kept on the transaction object itself rather than
// in a list, so it can't be mismatched.
const transactionIdsMap: Remapping = {};
const previousTransactionIds: Map<MinimalTransaction, number> = new Map();
for (let i = 0; i < world.transactions.length; i++) {
const tr = world.transactions[i];
tr.date = parseDate(tr.date);
tr.debitDate = parseDate(tr.debitDate);
tr.importDate = parseDate(tr.importDate);
if (tr.date === null) {
log.warn('Ignoring transaction without date\n', tr);
continue;
}
if (typeof tr.amount !== 'number' || isNaN(tr.amount)) {
log.warn('Ignoring transaction without valid amount\n', tr);
continue;
}
// Map transaction to account.
if (typeof tr.accountId !== 'undefined') {
if (!accountIdToAccount.has(tr.accountId)) {
log.warn('Ignoring orphan transaction:\n', tr);
continue;
}
tr.accountId = accountIdToAccount.get(tr.accountId);
} else {
if (!vendorToOwnAccountId.has(tr.bankAccount)) {
log.warn('Ignoring orphan transaction:\n', tr);
continue;
}
tr.accountId = vendorToOwnAccountId.get(tr.bankAccount);
}
// Remove bankAccount as the transaction is now linked to account with accountId prop.
delete tr.bankAccount;
const categoryId = tr.categoryId;
if (typeof categoryId !== 'undefined' && categoryId !== null) {
if (typeof categoryMap[categoryId] === 'undefined') {
log.warn('Unknown category, unsetting for transaction:\n', tr);
}
tr.categoryId = categoryMap[categoryId];
}
// Set transaction type base on transactionId.
// (Maintain 'operation' in name, as it's a deprecated field.)
if (typeof tr.operationTypeID !== 'undefined') {
const key = tr.operationTypeID.toString();
if (importedTypesMap.has(key)) {
tr.type = importedTypesMap.get(key);
} else {
tr.type = UNKNOWN_TRANSACTION_TYPE;
}
delete tr.operationTypeID;
}
// If there is no import date, set it to now.
if (tr.importDate === null) {
tr.importDate = new Date();
}
// If there is no label use the rawLabel, and vice-versa.
if (typeof tr.label === 'undefined') {
tr.label = tr.rawLabel;
}
if (typeof tr.rawLabel === 'undefined') {
tr.rawLabel = tr.label;
}
if (typeof tr.label === 'undefined' && typeof tr.rawLabel === 'undefined') {
log.warn('Ignoring transaction without label/rawLabel:\n', tr);
continue;
}
// Consider that old imports have the type set by the user, to have a consistent behaviour
// with the migration.
if (typeof tr.isUserDefinedType === 'undefined') {
tr.isUserDefinedType = true;
}
// Now that we are sure the transaction is to be imported (we are past all `continue`),
// store the id.
if (typeof tr.id === 'number') {
previousTransactionIds.set(tr, tr.id);
}
// Remove contents of deprecated fields, if there were any.
delete tr.attachments;
delete tr.binary;
// Remove ids that differ depending on the instance (due to auto-increment mainly).
delete tr.id;
delete tr.userId;
// Add the transaction to its account group.
const accountGroup = newByAccountId.get(tr.accountId) || [];
accountGroup.push(tr);
newByAccountId.set(tr.accountId, accountGroup);
}
for (const [accountId, provided] of newByAccountId) {
const known = await Transaction.byAccount(userId, accountId);
const diffResult = diffTransactions(known, provided, { perfectMatchMaxDateThreshold });
// Ignore perfect matches and knownOrphans. Only import the "provider" (import) orphans
// and the duplicate candidates, since we're not too sure about those.
const transactions = diffResult.providerOrphans;
const candidates = diffResult.duplicateCandidates.map(pair => pair[1]);
transactions.push(...candidates);
if (transactions.length > 0) {
log.info(`Importing ${transactions.length} transactions for account ${accountId}.`);
const insertedIds = await Transaction.bulkCreate(userId, transactions);
// There should be the same number of inserted transactions as transactions to create…
// Otherwise we can't map new ids to old ids.
if (transactions.length === insertedIds.length) {
insertedIds.forEach((trId, index) => {
const previousId = previousTransactionIds.get(transactions[index]);
if (typeof previousId === 'number') {
transactionIdsMap[previousId] = trId;
}
});
} else {
log.warn(
'Could not map the imported transaction ids to the inserted ones, the duplicates pairs to ignore will be lost.'
);
}
} else {
log.info(`No transactions to import for account ${accountId}.`);
}
}
log.info('Done.');
log.info('Import duplicates pairs to ignore...');
for (const pair of world.ignoredDuplicates) {
const newId = transactionIdsMap[pair[0]];
const newOtherId = transactionIdsMap[pair[1]];
if (typeof newId === 'number' && typeof newOtherId === 'number') {
await DuplicatesIgnored.create(userId, newId, newOtherId);
} else {
log.warn(
`Could not import pair of duplicates to ignore because the ids do not match the ids mapping: ${pair[0]}/${pair[1]}`
);
}
}
log.info('Done.');
log.info('Import settings...');
for (const setting of world.settings) {
if (ConfigGhostSettings.has(setting.key)) {
continue;
}
// Reset the default account id, if it's set.
if (
setting.key === DEFAULT_ACCOUNT_ID &&
setting.value !== DefaultSettings.get(DEFAULT_ACCOUNT_ID)
) {
const settingAsNumber = Number.parseInt(setting.value, 10);
if (!accountIdToAccount.has(settingAsNumber)) {
log.warn(`unknown default account id: ${setting.value}, skipping.`);
continue;
}
setting.value = accountIdToAccount.get(settingAsNumber);
await Setting.updateByKey(userId, DEFAULT_ACCOUNT_ID, setting.value);
continue;
}
// Overwrite the previous value of the demo-mode, if it was set.
if (setting.key === DEMO_MODE && setting.value === 'true') {
const found = await Setting.byKey(userId, DEMO_MODE);
if (found && found.value !== 'true') {
await Setting.updateByKey(userId, DEMO_MODE, 'true');
continue;
}
}
delete setting.userId;
// Note that former existing values are not overwritten!
await Setting.findOrCreateByKey(userId, setting.key, setting.value);
}
log.info('Done.');
log.info('Import alerts...');
const existingAlerts = await Alert.all(userId);
for (const a of world.alerts) {
// Map alert to account.
if (typeof a.accountId !== 'undefined') {
if (!accountIdToAccount.has(a.accountId)) {
log.warn('Ignoring orphan alert:\n', a);
continue;
}
a.accountId = accountIdToAccount.get(a.accountId);
} else {
if (!vendorToOwnAccountId.has(a.bankAccount)) {
log.warn('Ignoring orphan alert:\n', a);
continue;
}
a.accountId = vendorToOwnAccountId.get(a.bankAccount);
}
// Don't reimport an existing similar alert.
let foundDuplicate = false;
for (const known of existingAlerts) {
if (
known.accountId === a.accountId &&
known.type === a.type &&
known.frequency === a.frequency &&
known.limit === a.limit &&
known.order === a.order
) {
foundDuplicate = true;
break;
}
}
if (foundDuplicate) {
continue;
}
// Remove bankAccount as the alert is now linked to account with accountId prop.
delete a.bankAccount;
delete a.id;
delete a.userId;
await Alert.create(userId, a);
}
log.info('Done.');
log.info('Import transaction rules...');
const existingRules = new Set(
(await TransactionRule.allOrdered(userId)).map(r => TransactionRule.easyHash(r))
);
for (const r of world.transactionRules) {
// Clean up actions: they must refer to existing entities.
const removeActions = [];
let i = 0;
for (const a of r.actions) {
if (
typeof a.categoryId === 'undefined' ||
typeof categoryMap[a.categoryId] === 'undefined'
) {
log.warn('Ignoring unknown category action.');
removeActions.push(i);
} else {
a.categoryId = categoryMap[a.categoryId];
}
i += 1;
}
removeActions.reverse();
for (const j of removeActions) {
r.actions.splice(j, 1);
}
for (const a of r.actions) {
delete a.ruleId;
delete a.id;
// Manually fill the user id. Note this may clobber the previous
// one, if it was existing, which is fine: we should not rely on
// those in general.
a.userId = userId;
}
for (const c of r.conditions) {
delete c.ruleId;
delete c.id;
// See above.
c.userId = userId;
}
if (r.actions.length === 0) {
log.warn('Ignoring rule with no actions.');
return;
}
if (r.conditions.length === 0) {
log.warn('Ignoring rule with no conditions.');
return;
}
delete r.id;
delete r.userId;
if (!existingRules.has(TransactionRule.easyHash(r))) {
await TransactionRule.create(userId, r);
}
}
log.info('Done.');
log.info('Import recurring transactions...');
const currentDate = new Date();
const currentYear = currentDate.getFullYear();
const currentMonth = currentDate.getMonth();
const recurringTransactionsMap = new Map<number, number>();
const existingRecurringTransactionsHashMap = new Map<string, number>();
const existingRecurringTransactions = await RecurringTransaction.all(userId);
existingRecurringTransactions.forEach(rt => {
existingRecurringTransactionsHashMap.set(RecurringTransaction.easyHash(rt), rt.id);
});
for (let i = 0; i < world.recurringTransactions.length; i++) {
const rt = world.recurringTransactions[i];
if (typeof rt.amount !== 'number' || isNaN(rt.amount)) {
log.warn('Ignoring recurring transactions without valid amount\n', rt);
continue;
}
// Map recurring transaction to account.
if (typeof rt.accountId !== 'undefined') {
if (!accountIdToAccount.has(rt.accountId)) {
log.warn('Ignoring orphan recurring transaction:\n', rt);
continue;
}
rt.accountId = accountIdToAccount.get(rt.accountId);
}
if (typeof rt.label === 'undefined') {
log.warn('Ignoring recurring transaction without label/rawLabel:\n', rt);
continue;
}
const hash = RecurringTransaction.easyHash(rt);
const exists = existingRecurringTransactionsHashMap.has(hash);
if (exists) {
recurringTransactionsMap.set(rt.id, existingRecurringTransactionsHashMap.get(hash)!);
} else {
const created = await RecurringTransaction.create(userId, rt);
recurringTransactionsMap.set(rt.id, created.id);
}
}
for (let i = 0; i < world.appliedRecurringTransactions.length; i++) {
const art = world.appliedRecurringTransactions[i];
// Only import applied recurring transactions from current month.
if (art.year !== currentYear || art.month !== currentMonth) {
continue;
}
if (
!accountIdToAccount.has(art.accountId) ||
!recurringTransactionsMap.has(art.recurringTransactionId)
) {
log.warn('Ignoring orphan applied recurring transaction:\n', art);
continue;
}
art.accountId = accountIdToAccount.get(art.accountId);
art.recurringTransactionId = recurringTransactionsMap.get(art.recurringTransactionId);
const exists = await AppliedRecurringTransaction.exists(
userId,
art.accountId,
art.recurringTransactionId,
art.month,
art.year
);
if (!exists) {
await AppliedRecurringTransaction.create(userId, art);
}
}
log.info('Done.');
log.info('Apply banks migrations');
await runDataMigrations(userId);
log.info('Done.');
}
export async function import_(req: IdentifiedRequest<any>, res: express.Response) {
try {
const { id: userId } = req.user;
if (await isDemoEnabled(userId)) {
throw new KError("importing accesses isn't allowed in demo mode", 400);
}
if (!req.body.data) {
throw new KError('missing parameter "data" in the file', 400);
}
let world = req.body.data;
if (req.body.encrypted) {
if (typeof req.body.data !== 'string') {
throw new KError('content of an encrypted export should be an encoded string', 400);
}
if (typeof req.body.passphrase !== 'string') {
throw new KError('missing parameter "passphrase"', 400);
}
if (process.kresus.salt === null) {
throw new KError(
"server hasn't been configured for encryption; " +
'please ask your administrator to provide a salt'
);
}
world = decryptData(world, req.body.passphrase);
try {
world = JSON.parse(world);
} catch (err) {
throw new KError(
'Invalid JSON file or bad passphrase.',
400,
getErrorCode('INVALID_PASSWORD_JSON_EXPORT')
);
}
} else if (typeof req.body.data !== 'object') {
throw new KError('content of a JSON export should be a JSON object', 400);
}
await importData(userId, world);
log.info('Import finished with success!');
res.status(200).end();
} catch (err) {
asyncErr(res, err, 'when importing data');
}
}
export async function importOFX_(req: IdentifiedRequest<any>, res: express.Response) {
try {
const { id: userId } = req.user;
log.info('Parsing OFX file...');
const userData = JSON.parse(req.body);
const convertedData = await ofxToKresus(userData.data);
let dontCreateAccess = false;
// Set the accessId set by the user.
if (typeof userData.accessId === 'number' && convertedData) {
// Make sure the access exists.
if (!(await Access.exists(userId, userData.accessId))) {
throw new KError('No existing access for this access id', 400);
}
// Replace the accessId in the converted data by this one.
convertedData.accesses[0].id = userData.accessId;
convertedData.accounts.forEach(acc => {
acc.accessId = userData.accessId;
});
dontCreateAccess = true;
}
await importData(userId, convertedData, dontCreateAccess);
log.info('Import finished with success!');
res.status(200).end();
} catch (err) {
asyncErr(res, err, 'when importing data');
}
}
export const testing = {
ofxToKresus,
encryptData,
decryptData,
getAllData,
};