Refactor data handlers + misc fixes

This refactors the data handlers into a more standard/understood model-like architecture, to prepare for a new, more robust config system.

It also fixes a problem with creating new Notes and uploading new Photos via the web interface (related to #58).

Finally, it speeds up docker builds by sending in less files, which aren't necessary or will be built anyway.

This is all in preparation to allow building #13 more robustly.
This commit is contained in:
Bruno Bernardino
2025-05-24 08:24:10 +01:00
parent e1193a2770
commit 6cfb62d1a2
61 changed files with 1822 additions and 1774 deletions

View File

@@ -5,7 +5,7 @@ import 'std/dotenv/load.ts';
import { baseUrl, generateHash, isRunningLocally } from './utils/misc.ts';
import { User, UserSession } from './types.ts';
import { createUserSession, deleteUserSession, getUserByEmail, validateUserAndSession } from './data/user.ts';
import { UserModel, UserSessionModel, validateUserAndSession } from './models/user.ts';
import { isCookieDomainAllowed, isCookieDomainSecurityDisabled } from './config.ts';
const JWT_SECRET = Deno.env.get('JWT_SECRET') || '';
@@ -102,7 +102,7 @@ async function getDataFromAuthorizationHeader(authorizationHeader: string) {
const hashedPassword = await generateHash(`${basicAuthPassword}:${PASSWORD_SALT}`, 'SHA-256');
const user = await getUserByEmail(basicAuthUsername);
const user = await UserModel.getByEmail(basicAuthUsername);
if (!user || (user.hashed_password !== hashedPassword && user.extra.dav_hashed_password !== hashedPassword)) {
throw new Error('Email not found or invalid password.');
@@ -159,7 +159,7 @@ export async function logoutUser(request: Request) {
const { session_id } = tokenData;
// Delete user session
await deleteUserSession(session_id);
await UserSessionModel.delete(session_id);
// Generate response with empty and expiring cookie
const cookie: Cookie = {
@@ -210,7 +210,7 @@ export async function createSessionCookie(
response: Response,
isShortLived = false,
) {
const newSession = await createUserSession(user, isShortLived);
const newSession = await UserSessionModel.create(user, isShortLived);
// Generate response with session cookie
const token = await generateToken({ user_id: user.id, session_id: newSession.id });

View File

@@ -1,11 +1,11 @@
import 'std/dotenv/load.ts';
import { isThereAnAdmin } from './data/user.ts';
import { UserModel } from './models/user.ts';
export async function isSignupAllowed() {
const areSignupsAllowed = Deno.env.get('CONFIG_ALLOW_SIGNUPS') === 'true';
const areThereAdmins = await isThereAnAdmin();
const areThereAdmins = await UserModel.isThereAnAdmin();
if (areSignupsAllowed || !areThereAdmins) {
return true;

View File

@@ -1,42 +0,0 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import { Dashboard } from '/lib/types.ts';
const db = new Database();
export async function getDashboardByUserId(userId: string) {
const dashboard = (await db.query<Dashboard>(sql`SELECT * FROM "bewcloud_dashboards" WHERE "user_id" = $1 LIMIT 1`, [
userId,
]))[0];
return dashboard;
}
export async function createDashboard(userId: string) {
const data: Dashboard['data'] = { links: [], notes: '' };
const newDashboard = (await db.query<Dashboard>(
sql`INSERT INTO "bewcloud_dashboards" (
"user_id",
"data"
) VALUES ($1, $2)
RETURNING *`,
[
userId,
JSON.stringify(data),
],
))[0];
return newDashboard;
}
export async function updateDashboard(dashboard: Dashboard) {
await db.query(
sql`UPDATE "bewcloud_dashboards" SET
"data" = $2
WHERE "id" = $1`,
[
dashboard.id,
JSON.stringify(dashboard.data),
],
);
}

View File

@@ -1,469 +0,0 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { Budget, Expense } from '/lib/types.ts';
const db = new Database();
export async function getBudgets(
userId: string,
month: string,
{ skipRecalculation = false }: { skipRecalculation?: boolean } = {},
) {
if (!skipRecalculation) {
await recalculateMonthBudgets(userId, month);
}
const budgets = await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "month" = $2 ORDER BY cast("extra"->>'availableValue' as numeric) DESC, "value" DESC, "name" ASC`,
[
userId,
month,
],
);
// Numeric values come as strings, so we need to convert them to numbers
return budgets.map((budget) => ({
...budget,
value: Number(budget.value),
}));
}
export async function getBudgetByName(userId: string, month: string, name: string) {
const budget = (await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "month" = $2 AND LOWER("name") = LOWER($3)`,
[
userId,
month,
name,
],
))[0];
if (!budget) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...budget,
value: Number(budget.value),
};
}
export async function getBudgetById(userId: string, id: string) {
const budget = (await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "id" = $2`,
[userId, id],
))[0];
if (!budget) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...budget,
value: Number(budget.value),
};
}
export async function getAllBudgetsForExport(
userId: string,
): Promise<(Omit<Budget, 'id' | 'user_id' | 'created_at' | 'extra'> & { extra: Record<never, never> })[]> {
const budgets = await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 ORDER BY "month" DESC, "name" ASC`,
[
userId,
],
);
return budgets.map((budget) => ({
name: budget.name,
month: budget.month,
// Numeric values come as strings, so we need to convert them to numbers
value: Number(budget.value),
extra: {},
}));
}
export async function getExpenses(userId: string, month: string) {
const expenses = await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND "date" >= $2 AND "date" <= $3 ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
`${month}-01`,
`${month}-31`,
],
);
// Numeric values come as strings, so we need to convert them to numbers
return expenses.map((expense) => ({
...expense,
cost: Number(expense.cost),
}));
}
export async function getExpenseByName(userId: string, name: string) {
const expense = (await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND LOWER("description") = LOWER($2) ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
name,
],
))[0];
if (!expense) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...expense,
cost: Number(expense.cost),
};
}
export async function getExpenseSuggestions(userId: string, name: string) {
const expenses = await db.query<Pick<Expense, 'description'>>(
sql`SELECT "description" FROM "bewcloud_expenses" WHERE "user_id" = $1 AND LOWER("description") ILIKE LOWER($2) GROUP BY "description" ORDER BY LENGTH("description") ASC, "description" ASC`,
[
userId,
`%${name}%`,
],
);
return expenses.map((expense) => expense.description);
}
export async function getAllExpensesForExport(
userId: string,
): Promise<(Omit<Expense, 'id' | 'user_id' | 'created_at' | 'extra'> & { extra: Record<never, never> })[]> {
const expenses = await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
],
);
return expenses.map((expense) => ({
description: expense.description,
budget: expense.budget,
date: expense.date,
is_recurring: expense.is_recurring,
// Numeric values come as strings, so we need to convert them to numbers
cost: Number(expense.cost),
extra: {},
}));
}
export async function getExpenseById(userId: string, id: string) {
const expense = (await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND "id" = $2`,
[userId, id],
))[0];
if (!expense) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...expense,
cost: Number(expense.cost),
};
}
export async function createBudget(userId: string, name: string, month: string, value: number) {
const extra: Budget['extra'] = {
usedValue: 0,
availableValue: value,
};
const newBudget = (await db.query<Budget>(
sql`INSERT INTO "bewcloud_budgets" (
"user_id",
"name",
"month",
"value",
"extra"
) VALUES ($1, $2, $3, $4, $5)
RETURNING *`,
[
userId,
name,
month,
value,
JSON.stringify(extra),
],
))[0];
// Numeric values come as strings, so we need to convert them to numbers
return {
...newBudget,
value: Number(newBudget.value),
};
}
export async function updateBudget(
budget: Budget,
{ skipRecalculation = false }: { skipRecalculation?: boolean } = {},
) {
await db.query(
sql`UPDATE "bewcloud_budgets" SET
"name" = $2,
"month" = $3,
"value" = $4,
"extra" = $5
WHERE "id" = $1`,
[
budget.id,
budget.name,
budget.month,
budget.value,
JSON.stringify(budget.extra),
],
);
if (!skipRecalculation) {
await recalculateMonthBudgets(budget.user_id, budget.month);
}
}
export async function deleteBudget(userId: string, id: string) {
await db.query(
sql`DELETE FROM "bewcloud_budgets" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
}
export async function deleteAllBudgetsAndExpenses(userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_expenses" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_budgets" WHERE "user_id" = $1`,
[
userId,
],
);
}
export async function createExpense(
userId: string,
cost: number,
description: string,
budget: string,
date: string,
is_recurring: boolean,
{ skipRecalculation = false, skipBudgetMatching = false, skipBudgetCreation = false }: {
skipRecalculation?: boolean;
skipBudgetMatching?: boolean;
skipBudgetCreation?: boolean;
} = {},
) {
const extra: Expense['extra'] = {};
if (!budget.trim()) {
budget = 'Misc';
}
// Match budget to an existing expense "by default"
if (!skipBudgetMatching && budget === 'Misc') {
const existingExpense = await getExpenseByName(userId, description);
if (existingExpense) {
budget = existingExpense.budget;
}
}
if (!skipBudgetCreation) {
const existingBudgetInMonth = await getBudgetByName(userId, date.substring(0, 7), budget);
if (!existingBudgetInMonth) {
await createBudget(userId, budget, date.substring(0, 7), 100);
}
}
const newExpense = (await db.query<Expense>(
sql`INSERT INTO "bewcloud_expenses" (
"user_id",
"cost",
"description",
"budget",
"date",
"is_recurring",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *`,
[
userId,
cost,
description,
budget,
date,
is_recurring,
JSON.stringify(extra),
],
))[0];
if (!skipRecalculation) {
await recalculateMonthBudgets(userId, date.substring(0, 7));
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...newExpense,
cost: Number(newExpense.cost),
};
}
export async function updateExpense(expense: Expense) {
const existingBudgetInMonth = await getBudgetByName(expense.user_id, expense.date.substring(0, 7), expense.budget);
if (!existingBudgetInMonth) {
await createBudget(expense.user_id, expense.budget, expense.date.substring(0, 7), 100);
}
await db.query(
sql`UPDATE "bewcloud_expenses" SET
"cost" = $2,
"description" = $3,
"budget" = $4,
"date" = $5,
"is_recurring" = $6,
"extra" = $7
WHERE "id" = $1`,
[
expense.id,
expense.cost,
expense.description,
expense.budget,
expense.date,
expense.is_recurring,
JSON.stringify(expense.extra),
],
);
await recalculateMonthBudgets(expense.user_id, expense.date.substring(0, 7));
}
export async function deleteExpense(userId: string, id: string) {
const expense = await getExpenseById(userId, id);
await db.query(
sql`DELETE FROM "bewcloud_expenses" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await recalculateMonthBudgets(userId, expense!.date.substring(0, 7));
}
export async function generateMonthlyBudgetsAndExpenses(userId: string, month: string) {
const lock = new Locker(`expenses:${userId}:${month}`);
await lock.acquire();
let addedBudgetsCount = 0;
let addedExpensesCount = 0;
try {
// Confirm there are no budgets or expenses for this month
const monthBudgets = await getBudgets(userId, month, { skipRecalculation: true });
const monthExpenses = await getExpenses(userId, month);
if (monthBudgets.length > 0 || monthExpenses.length > 0) {
throw new Error('Budgets and expenses already exist for this month!');
}
// Get the previous month's budgets, to copy over
const previousMonthDate = new Date(month);
previousMonthDate.setMonth(previousMonthDate.getMonth() - 1);
const previousMonth = previousMonthDate.toISOString().substring(0, 7);
const budgets = await getBudgets(userId, previousMonth, { skipRecalculation: true });
for (const budget of budgets) {
await createBudget(userId, budget.name, month, budget.value);
addedBudgetsCount++;
}
// Get the recurring expenses for the previous month, to copy over
const recurringExpenses = (await getExpenses(userId, previousMonth)).filter((expense) => expense.is_recurring);
for (const expense of recurringExpenses) {
await createExpense(
userId,
expense.cost,
expense.description,
expense.budget,
expense.date.replace(previousMonth, month),
expense.is_recurring,
{ skipRecalculation: true },
);
addedExpensesCount++;
}
console.info(`Added ${addedBudgetsCount} new budgets and ${addedExpensesCount} new expenses for ${month}`);
lock.release();
} catch (error) {
lock.release();
throw error;
}
}
export async function recalculateMonthBudgets(userId: string, month: string) {
const lock = new Locker(`expenses:${userId}:${month}`);
await lock.acquire();
try {
const budgets = await getBudgets(userId, month, { skipRecalculation: true });
const expenses = await getExpenses(userId, month);
// Calculate total expenses for each budget
const budgetExpenses = new Map<string, number>();
for (const expense of expenses) {
const currentTotal = Number(budgetExpenses.get(expense.budget) || 0);
budgetExpenses.set(expense.budget, Number(currentTotal + expense.cost));
}
// Update each budget with new calculations
for (const budget of budgets) {
const usedValue = Number(budgetExpenses.get(budget.name) || 0);
const availableValue = Number(budget.value - usedValue);
const updatedBudget: Budget = {
...budget,
extra: {
...budget.extra,
usedValue,
availableValue,
},
};
if (budget.extra.usedValue !== usedValue || budget.extra.availableValue !== availableValue) {
await updateBudget(updatedBudget, { skipRecalculation: true });
}
}
lock.release();
} catch (error) {
lock.release();
throw error;
}
}

View File

@@ -1,504 +0,0 @@
import { join } from 'std/path/join.ts';
import { resolve } from 'std/path/resolve.ts';
import { lookup } from 'mrmime';
import { getFilesRootPath } from '/lib/config.ts';
import { Directory, DirectoryFile } from '/lib/types.ts';
import { sortDirectoriesByName, sortEntriesByName, sortFilesByName, TRASH_PATH } from '/lib/utils/files.ts';
/**
* Ensures the user path is valid and securely accessible (meaning it's not trying to access files outside of the user's root directory).
* Does not check if the path exists.
*
* @param userId - The user ID
* @param path - The relative path (user-provided) to check
*/
export function ensureUserPathIsValidAndSecurelyAccessible(userId: string, path: string): void {
const userRootPath = join(getFilesRootPath(), userId, '/');
const fullPath = join(userRootPath, path);
const resolvedFullPath = `${resolve(fullPath)}/`;
if (!resolvedFullPath.startsWith(userRootPath)) {
throw new Error('Invalid file path');
}
}
export async function getDirectories(userId: string, path: string): Promise<Directory[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
const directories: Directory[] = [];
const directoryEntries = (await getPathEntries(userId, path)).filter((entry) => entry.isDirectory || entry.isSymlink);
for (const entry of directoryEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const directory: Directory = {
user_id: userId,
parent_path: path,
directory_name: entry.name,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
directories.push(directory);
}
directories.sort(sortDirectoriesByName);
return directories;
}
export async function getFiles(userId: string, path: string): Promise<DirectoryFile[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
const files: DirectoryFile[] = [];
const fileEntries = (await getPathEntries(userId, path)).filter((entry) => entry.isFile);
for (const entry of fileEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const file: DirectoryFile = {
user_id: userId,
parent_path: path,
file_name: entry.name,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
files.sort(sortFilesByName);
return files;
}
async function getPathEntries(userId: string, path: string): Promise<Deno.DirEntry[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
// Ensure the user directory exists
if (path === '/') {
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(join(rootPath, TRASH_PATH), { recursive: true });
}
}
}
// Ensure the Notes or Photos directories exist, if being requested
if (path === '/Notes/' || path === '/Photos/') {
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(rootPath, { recursive: true });
}
}
}
const entries: Deno.DirEntry[] = [];
for await (const dirEntry of Deno.readDir(rootPath)) {
entries.push(dirEntry);
}
entries.sort(sortEntriesByName);
return entries;
}
export async function createDirectory(userId: string, path: string, name: string): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
await Deno.mkdir(join(rootPath, name), { recursive: true });
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function renameDirectoryOrFile(
userId: string,
oldPath: string,
newPath: string,
oldName: string,
newName: string,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(oldPath, oldName));
ensureUserPathIsValidAndSecurelyAccessible(userId, join(newPath, newName));
const oldRootPath = join(getFilesRootPath(), userId, oldPath);
const newRootPath = join(getFilesRootPath(), userId, newPath);
try {
await Deno.rename(join(oldRootPath, oldName), join(newRootPath, newName));
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function deleteDirectoryOrFile(userId: string, path: string, name: string): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
if (path.startsWith(TRASH_PATH)) {
await Deno.remove(join(rootPath, name), { recursive: true });
} else {
const trashPath = join(getFilesRootPath(), userId, TRASH_PATH);
await Deno.rename(join(rootPath, name), join(trashPath, name));
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function createFile(
userId: string,
path: string,
name: string,
contents: string | ArrayBuffer,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
// Ensure the directory exist, if being requested
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(rootPath, { recursive: true });
}
}
if (typeof contents === 'string') {
await Deno.writeTextFile(join(rootPath, name), contents, { append: false, createNew: true });
} else {
await Deno.writeFile(join(rootPath, name), new Uint8Array(contents), { append: false, createNew: true });
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function updateFile(
userId: string,
path: string,
name: string,
contents: string,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
await Deno.writeTextFile(join(rootPath, name), contents, { append: false, createNew: false });
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function getFile(
userId: string,
path: string,
name?: string,
): Promise<{ success: boolean; contents?: Uint8Array; contentType?: string; byteSize?: number }> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name || ''));
const rootPath = join(getFilesRootPath(), userId, path);
try {
const stat = await Deno.stat(join(rootPath, name || ''));
if (stat) {
const contents = await Deno.readFile(join(rootPath, name || ''));
const extension = (name || path).split('.').slice(-1).join('').toLowerCase();
const contentType = lookup(extension) || 'application/octet-stream';
return {
success: true,
contents,
contentType,
byteSize: stat.size,
};
}
} catch (error) {
console.error(error);
}
return {
success: false,
};
}
export async function searchFilesAndDirectories(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; directories: Directory[]; files: DirectoryFile[] }> {
const directoryNamesResult = await searchDirectoryNames(userId, searchTerm);
const fileNamesResult = await searchFileNames(userId, searchTerm);
const fileContentsResult = await searchFileContents(userId, searchTerm);
const success = directoryNamesResult.success && fileNamesResult.success && fileContentsResult.success;
const directories = [...directoryNamesResult.directories];
directories.sort(sortDirectoriesByName);
const files = [...fileNamesResult.files, ...fileContentsResult.files];
files.sort(sortFilesByName);
return {
success,
directories,
files,
};
}
async function searchDirectoryNames(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; directories: Directory[] }> {
const rootPath = join(getFilesRootPath(), userId);
const directories: Directory[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`find`, {
args: [
`.`, // proper cwd is sent below
`-type`,
`d,l`, // directories and symbolic links
`-iname`,
`*${searchTerm}*`,
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code !== 0) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "find"`);
}
const output = new TextDecoder().decode(stdout);
const matchingDirectories = output.split('\n').map((directoryPath) => directoryPath.trim()).filter(Boolean);
for (const relativeDirectoryPath of matchingDirectories) {
const stat = await Deno.stat(join(rootPath, relativeDirectoryPath));
let parentPath = `/${relativeDirectoryPath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const directoryName = relativeDirectoryPath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const directory: Directory = {
user_id: userId,
parent_path: parentPath,
directory_name: directoryName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
directories.push(directory);
}
return { success: true, directories };
} catch (error) {
console.error(error);
}
return { success: false, directories };
}
async function searchFileNames(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; files: DirectoryFile[] }> {
const rootPath = join(getFilesRootPath(), userId);
const files: DirectoryFile[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`find`, {
args: [
`.`, // proper cwd is sent below
`-type`,
`f`,
`-iname`,
`*${searchTerm}*`,
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code !== 0) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "find"`);
}
const output = new TextDecoder().decode(stdout);
const matchingFiles = output.split('\n').map((filePath) => filePath.trim()).filter(Boolean);
for (const relativeFilePath of matchingFiles) {
const stat = await Deno.stat(join(rootPath, relativeFilePath));
let parentPath = `/${relativeFilePath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const fileName = relativeFilePath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const file: DirectoryFile = {
user_id: userId,
parent_path: parentPath,
file_name: fileName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
return { success: true, files };
} catch (error) {
console.error(error);
}
return { success: false, files };
}
async function searchFileContents(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; files: DirectoryFile[] }> {
const rootPath = join(getFilesRootPath(), userId);
const files: DirectoryFile[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`grep`, {
args: [
`-rHisl`,
`${searchTerm}`,
`.`, // proper cwd is sent below
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code > 1) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "grep"`);
}
const output = new TextDecoder().decode(stdout);
const matchingFiles = output.split('\n').map((filePath) => filePath.trim()).filter(Boolean);
for (const relativeFilePath of matchingFiles) {
const stat = await Deno.stat(join(rootPath, relativeFilePath));
let parentPath = `/${relativeFilePath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const fileName = relativeFilePath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const file: DirectoryFile = {
user_id: userId,
parent_path: parentPath,
file_name: fileName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
return { success: true, files };
} catch (error) {
console.error(error);
}
return { success: false, files };
}

View File

@@ -1,311 +0,0 @@
import { Feed } from 'https://deno.land/x/rss@1.0.0/mod.ts';
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { NewsFeed, NewsFeedArticle } from '/lib/types.ts';
import {
findFeedInUrl,
getArticleUrl,
getFeedInfo,
JsonFeed,
parseTextFromHtml,
parseUrl,
parseUrlAsGooglebot,
parseUrlWithProxy,
} from '/lib/feed.ts';
const db = new Database();
export async function getNewsFeeds(userId: string) {
const newsFeeds = await db.query<NewsFeed>(sql`SELECT * FROM "bewcloud_news_feeds" WHERE "user_id" = $1`, [
userId,
]);
return newsFeeds;
}
export async function getNewsFeed(id: string, userId: string) {
const newsFeeds = await db.query<NewsFeed>(
sql`SELECT * FROM "bewcloud_news_feeds" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return newsFeeds[0];
}
export async function getNewsArticles(userId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "user_id" = $1 ORDER BY "article_date" DESC`,
[
userId,
],
);
return articles;
}
export async function getNewsArticlesByFeedId(feedId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "feed_id" = $1 ORDER BY "article_date" DESC`,
[
feedId,
],
);
return articles;
}
export async function getNewsArticle(id: string, userId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return articles[0];
}
export async function createNewsFeed(userId: string, feedUrl: string) {
const extra: NewsFeed['extra'] = {};
const newNewsFeed = (await db.query<NewsFeed>(
sql`INSERT INTO "bewcloud_news_feeds" (
"user_id",
"feed_url",
"extra"
) VALUES ($1, $2, $3)
RETURNING *`,
[
userId,
feedUrl,
JSON.stringify(extra),
],
))[0];
return newNewsFeed;
}
export async function updateNewsFeed(newsFeed: NewsFeed) {
await db.query(
sql`UPDATE "bewcloud_news_feeds" SET
"feed_url" = $2,
"last_crawled_at" = $3,
"extra" = $4
WHERE "id" = $1`,
[
newsFeed.id,
newsFeed.feed_url,
newsFeed.last_crawled_at,
JSON.stringify(newsFeed.extra),
],
);
}
export async function deleteNewsFeed(id: string, userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_news_feed_articles" WHERE "feed_id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feeds" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
}
export async function createsNewsArticle(
userId: string,
feedId: string,
article: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>,
) {
const extra: NewsFeedArticle['extra'] = {};
const newNewsArticle = (await db.query<NewsFeedArticle>(
sql`INSERT INTO "bewcloud_news_feed_articles" (
"user_id",
"feed_id",
"article_url",
"article_title",
"article_summary",
"article_date",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *`,
[
userId,
feedId,
article.article_url,
article.article_title,
article.article_summary,
article.article_date,
JSON.stringify(extra),
],
))[0];
return newNewsArticle;
}
export async function updateNewsArticle(article: NewsFeedArticle) {
await db.query(
sql`UPDATE "bewcloud_news_feed_articles" SET
"is_read" = $2,
"extra" = $3
WHERE "id" = $1`,
[
article.id,
article.is_read,
JSON.stringify(article.extra),
],
);
}
export async function markAllArticlesRead(userId: string) {
await db.query(
sql`UPDATE "bewcloud_news_feed_articles" SET
"is_read" = TRUE
WHERE "user_id" = $1`,
[
userId,
],
);
}
async function fetchNewsArticles(newsFeed: NewsFeed): Promise<Feed['entries'] | JsonFeed['items']> {
try {
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
throw new Error('Invalid News Feed!');
}
let feed: JsonFeed | Feed | null = null;
if (newsFeed.extra.crawl_type === 'direct') {
feed = await parseUrl(newsFeed.feed_url);
} else if (newsFeed.extra.crawl_type === 'googlebot') {
feed = await parseUrlAsGooglebot(newsFeed.feed_url);
} else if (newsFeed.extra.crawl_type === 'proxy') {
feed = await parseUrlWithProxy(newsFeed.feed_url);
}
return (feed as Feed)?.entries || (feed as JsonFeed)?.items || [];
} catch (error) {
console.error('Failed parsing feed to get articles', newsFeed.feed_url);
console.error(error);
}
return [];
}
type FeedArticle = Feed['entries'][number];
type JsonFeedArticle = JsonFeed['items'][number];
const MAX_ARTICLES_CRAWLED_PER_RUN = 10;
export async function crawlNewsFeed(newsFeed: NewsFeed) {
const lock = new Locker(`feeds:${newsFeed.id}`);
await lock.acquire();
try {
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
const feedUrl = await findFeedInUrl(newsFeed.feed_url);
if (!feedUrl) {
throw new Error(
`Invalid URL for feed: "${feedUrl}"`,
);
}
if (feedUrl !== newsFeed.feed_url) {
newsFeed.feed_url = feedUrl;
}
const feedInfo = await getFeedInfo(newsFeed.feed_url);
newsFeed.extra.title = feedInfo.title;
newsFeed.extra.feed_type = feedInfo.feed_type;
newsFeed.extra.crawl_type = feedInfo.crawl_type;
}
const feedArticles = await fetchNewsArticles(newsFeed);
const articles: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>[] = [];
for (const feedArticle of feedArticles) {
// Don't add too many articles per run
if (articles.length >= MAX_ARTICLES_CRAWLED_PER_RUN) {
continue;
}
const url = (feedArticle as JsonFeedArticle).url || getArticleUrl((feedArticle as FeedArticle).links) ||
feedArticle.id;
const articleIsoDate = (feedArticle as JsonFeedArticle).date_published ||
(feedArticle as FeedArticle).published?.toISOString() || (feedArticle as JsonFeedArticle).date_modified ||
(feedArticle as FeedArticle).updated?.toISOString();
const articleDate = articleIsoDate ? new Date(articleIsoDate) : new Date();
const summary = await parseTextFromHtml(
(feedArticle as FeedArticle).description?.value || (feedArticle as FeedArticle).content?.value ||
(feedArticle as JsonFeedArticle).content_text || (feedArticle as JsonFeedArticle).content_html ||
(feedArticle as JsonFeedArticle).summary || '',
);
if (url) {
articles.push({
article_title: (feedArticle as FeedArticle).title?.value || (feedArticle as JsonFeedArticle).title ||
url.replace('http://', '').replace('https://', ''),
article_url: url,
article_summary: summary,
article_date: articleDate,
});
}
}
const existingArticles = await getNewsArticlesByFeedId(newsFeed.id);
const existingArticleUrls = new Set<string>(existingArticles.map((article) => article.article_url));
const previousLatestArticleUrl = existingArticles[0]?.article_url;
let seenPreviousLatestArticleUrl = false;
let addedArticlesCount = 0;
for (const article of articles) {
// Stop looking after seeing the previous latest article
if (article.article_url === previousLatestArticleUrl) {
seenPreviousLatestArticleUrl = true;
}
if (!seenPreviousLatestArticleUrl && !existingArticleUrls.has(article.article_url)) {
try {
await createsNewsArticle(newsFeed.user_id, newsFeed.id, article);
++addedArticlesCount;
} catch (error) {
console.error(error);
console.error(`Failed to add new article: "${article.article_url}"`);
}
}
}
console.info('Added', addedArticlesCount, 'new articles');
newsFeed.last_crawled_at = new Date();
await updateNewsFeed(newsFeed);
lock.release();
} catch (error) {
lock.release();
throw error;
}
}

View File

@@ -1,286 +0,0 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import { User, UserSession, VerificationCode } from '/lib/types.ts';
import { generateRandomCode } from '/lib/utils/misc.ts';
import { isEmailEnabled, isForeverSignupEnabled } from '/lib/config.ts';
const db = new Database();
export async function isThereAnAdmin() {
const user =
(await db.query<User>(sql`SELECT * FROM "bewcloud_users" WHERE ("extra" ->> 'is_admin')::boolean IS TRUE LIMIT 1`))[
0
];
return Boolean(user);
}
export async function getUserByEmail(email: string) {
const lowercaseEmail = email.toLowerCase().trim();
const user = (await db.query<User>(sql`SELECT * FROM "bewcloud_users" WHERE "email" = $1 LIMIT 1`, [
lowercaseEmail,
]))[0];
return user;
}
export async function getUserById(id: string) {
const user = (await db.query<User>(sql`SELECT * FROM "bewcloud_users" WHERE "id" = $1 LIMIT 1`, [
id,
]))[0];
return user;
}
export async function createUser(email: User['email'], hashedPassword: User['hashed_password']) {
const trialDays = isForeverSignupEnabled() ? 36_525 : 30;
const now = new Date();
const trialEndDate = new Date(new Date().setUTCDate(new Date().getUTCDate() + trialDays));
const subscription: User['subscription'] = {
external: {},
expires_at: trialEndDate.toISOString(),
updated_at: now.toISOString(),
};
const extra: User['extra'] = { is_email_verified: isEmailEnabled() ? false : true };
// First signup will be an admin "forever"
if (!(await isThereAnAdmin())) {
extra.is_admin = true;
subscription.expires_at = new Date('2100-12-31').toISOString();
}
const newUser = (await db.query<User>(
sql`INSERT INTO "bewcloud_users" (
"email",
"subscription",
"status",
"hashed_password",
"extra"
) VALUES ($1, $2, $3, $4, $5)
RETURNING *`,
[
email,
JSON.stringify(subscription),
extra.is_admin || isForeverSignupEnabled() ? 'active' : 'trial',
hashedPassword,
JSON.stringify(extra),
],
))[0];
return newUser;
}
export async function updateUser(user: User) {
await db.query(
sql`UPDATE "bewcloud_users" SET
"email" = $2,
"subscription" = $3,
"status" = $4,
"hashed_password" = $5,
"extra" = $6
WHERE "id" = $1`,
[
user.id,
user.email,
JSON.stringify(user.subscription),
user.status,
user.hashed_password,
JSON.stringify(user.extra),
],
);
}
export async function deleteUser(userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_user_sessions" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_verification_codes" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feed_articles" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feeds" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_users" WHERE "id" = $1`,
[
userId,
],
);
}
export async function getSessionById(id: string) {
const session = (await db.query<UserSession>(
sql`SELECT * FROM "bewcloud_user_sessions" WHERE "id" = $1 AND "expires_at" > now() LIMIT 1`,
[
id,
],
))[0];
return session;
}
export async function createUserSession(user: User, isShortLived = false) {
const oneMonthFromToday = new Date(new Date().setUTCMonth(new Date().getUTCMonth() + 1));
const oneWeekFromToday = new Date(new Date().setUTCDate(new Date().getUTCDate() + 7));
const newSession: Omit<UserSession, 'id' | 'created_at'> = {
user_id: user.id,
expires_at: isShortLived ? oneWeekFromToday : oneMonthFromToday,
last_seen_at: new Date(),
};
const newUserSessionResult = (await db.query<UserSession>(
sql`INSERT INTO "bewcloud_user_sessions" (
"user_id",
"expires_at",
"last_seen_at"
) VALUES ($1, $2, $3)
RETURNING *`,
[
newSession.user_id,
newSession.expires_at,
newSession.last_seen_at,
],
))[0];
return newUserSessionResult;
}
export async function updateSession(session: UserSession) {
await db.query(
sql`UPDATE "bewcloud_user_sessions" SET
"expires_at" = $2,
"last_seen_at" = $3
WHERE "id" = $1`,
[
session.id,
session.expires_at,
session.last_seen_at,
],
);
}
export async function deleteUserSession(sessionId: string) {
await db.query(
sql`DELETE FROM "bewcloud_user_sessions" WHERE "id" = $1`,
[
sessionId,
],
);
}
export async function validateUserAndSession(userId: string, sessionId: string) {
const user = await getUserById(userId);
if (!user) {
throw new Error('Not Found');
}
const session = await getSessionById(sessionId);
if (!session || session.user_id !== user.id) {
throw new Error('Not Found');
}
const oneMonthFromToday = new Date(new Date().setUTCMonth(new Date().getUTCMonth() + 1));
session.last_seen_at = new Date();
session.expires_at = oneMonthFromToday;
await updateSession(session);
return { user, session };
}
export async function createVerificationCode(
user: User,
verificationId: string,
type: VerificationCode['verification']['type'],
) {
const inThirtyMinutes = new Date(new Date().setUTCMinutes(new Date().getUTCMinutes() + 30));
const code = generateRandomCode();
const newVerificationCode: Omit<VerificationCode, 'id' | 'created_at'> = {
user_id: user.id,
code,
expires_at: inThirtyMinutes,
verification: {
id: verificationId,
type,
},
};
await db.query(
sql`INSERT INTO "bewcloud_verification_codes" (
"user_id",
"code",
"expires_at",
"verification"
) VALUES ($1, $2, $3, $4)
RETURNING "id"`,
[
newVerificationCode.user_id,
newVerificationCode.code,
newVerificationCode.expires_at,
JSON.stringify(newVerificationCode.verification),
],
);
return code;
}
export async function validateVerificationCode(
user: User,
verificationId: string,
code: string,
type: VerificationCode['verification']['type'],
) {
const verificationCode = (await db.query<VerificationCode>(
sql`SELECT * FROM "bewcloud_verification_codes"
WHERE "user_id" = $1 AND
"code" = $2 AND
"verification" ->> 'type' = $3 AND
"verification" ->> 'id' = $4 AND
"expires_at" > now()
LIMIT 1`,
[
user.id,
code,
type,
verificationId,
],
))[0];
if (verificationCode) {
await db.query(
sql`DELETE FROM "bewcloud_verification_codes" WHERE "id" = $1`,
[
verificationCode.id,
],
);
} else {
throw new Error('Not Found');
}
}

View File

@@ -1,4 +1,4 @@
import { Client } from 'https://deno.land/x/postgres@v0.19.2/mod.ts';
import { Client } from 'https://deno.land/x/postgres@v0.19.3/mod.ts';
import 'std/dotenv/load.ts';
const POSTGRESQL_HOST = Deno.env.get('POSTGRESQL_HOST') || '';

45
lib/models/dashboard.ts Normal file
View File

@@ -0,0 +1,45 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import { Dashboard } from '/lib/types.ts';
const db = new Database();
export class DashboardModel {
static async getByUserId(userId: string) {
const dashboard =
(await db.query<Dashboard>(sql`SELECT * FROM "bewcloud_dashboards" WHERE "user_id" = $1 LIMIT 1`, [
userId,
]))[0];
return dashboard;
}
static async create(userId: string) {
const data: Dashboard['data'] = { links: [], notes: '' };
const newDashboard = (await db.query<Dashboard>(
sql`INSERT INTO "bewcloud_dashboards" (
"user_id",
"data"
) VALUES ($1, $2)
RETURNING *`,
[
userId,
JSON.stringify(data),
],
))[0];
return newDashboard;
}
static async update(dashboard: Dashboard) {
await db.query(
sql`UPDATE "bewcloud_dashboards" SET
"data" = $2
WHERE "id" = $1`,
[
dashboard.id,
JSON.stringify(dashboard.data),
],
);
}
}

479
lib/models/expenses.ts Normal file
View File

@@ -0,0 +1,479 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { Budget, Expense } from '/lib/types.ts';
const db = new Database();
export class BudgetModel {
static async list(
userId: string,
month: string,
{ skipRecalculation = false }: { skipRecalculation?: boolean } = {},
) {
if (!skipRecalculation) {
await recalculateMonthBudgets(userId, month);
}
const budgets = await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "month" = $2 ORDER BY cast("extra"->>'availableValue' as numeric) DESC, "value" DESC, "name" ASC`,
[
userId,
month,
],
);
// Numeric values come as strings, so we need to convert them to numbers
return budgets.map((budget) => ({
...budget,
value: Number(budget.value),
}));
}
static async getByName(userId: string, month: string, name: string) {
const budget = (await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "month" = $2 AND LOWER("name") = LOWER($3)`,
[
userId,
month,
name,
],
))[0];
if (!budget) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...budget,
value: Number(budget.value),
};
}
static async getById(userId: string, id: string) {
const budget = (await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 AND "id" = $2`,
[userId, id],
))[0];
if (!budget) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...budget,
value: Number(budget.value),
};
}
static async getAllForExport(
userId: string,
): Promise<(Omit<Budget, 'id' | 'user_id' | 'created_at' | 'extra'> & { extra: Record<never, never> })[]> {
const budgets = await db.query<Budget>(
sql`SELECT * FROM "bewcloud_budgets" WHERE "user_id" = $1 ORDER BY "month" DESC, "name" ASC`,
[
userId,
],
);
return budgets.map((budget) => ({
name: budget.name,
month: budget.month,
// Numeric values come as strings, so we need to convert them to numbers
value: Number(budget.value),
extra: {},
}));
}
static async create(userId: string, name: string, month: string, value: number) {
const extra: Budget['extra'] = {
usedValue: 0,
availableValue: value,
};
const newBudget = (await db.query<Budget>(
sql`INSERT INTO "bewcloud_budgets" (
"user_id",
"name",
"month",
"value",
"extra"
) VALUES ($1, $2, $3, $4, $5)
RETURNING *`,
[
userId,
name,
month,
value,
JSON.stringify(extra),
],
))[0];
// Numeric values come as strings, so we need to convert them to numbers
return {
...newBudget,
value: Number(newBudget.value),
};
}
static async update(
budget: Budget,
{ skipRecalculation = false }: { skipRecalculation?: boolean } = {},
) {
await db.query(
sql`UPDATE "bewcloud_budgets" SET
"name" = $2,
"month" = $3,
"value" = $4,
"extra" = $5
WHERE "id" = $1`,
[
budget.id,
budget.name,
budget.month,
budget.value,
JSON.stringify(budget.extra),
],
);
if (!skipRecalculation) {
await recalculateMonthBudgets(budget.user_id, budget.month);
}
}
static async delete(userId: string, id: string) {
await db.query(
sql`DELETE FROM "bewcloud_budgets" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
}
}
export class ExpenseModel {
static async list(userId: string, month: string) {
const expenses = await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND "date" >= $2 AND "date" <= $3 ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
`${month}-01`,
`${month}-31`,
],
);
// Numeric values come as strings, so we need to convert them to numbers
return expenses.map((expense) => ({
...expense,
cost: Number(expense.cost),
}));
}
static async getByName(userId: string, name: string) {
const expense = (await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND LOWER("description") = LOWER($2) ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
name,
],
))[0];
if (!expense) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...expense,
cost: Number(expense.cost),
};
}
static async listSuggestions(userId: string, name: string) {
const expenses = await db.query<Pick<Expense, 'description'>>(
sql`SELECT "description" FROM "bewcloud_expenses" WHERE "user_id" = $1 AND LOWER("description") ILIKE LOWER($2) GROUP BY "description" ORDER BY LENGTH("description") ASC, "description" ASC`,
[
userId,
`%${name}%`,
],
);
return expenses.map((expense) => expense.description);
}
static async getAllForExport(
userId: string,
): Promise<(Omit<Expense, 'id' | 'user_id' | 'created_at' | 'extra'> & { extra: Record<never, never> })[]> {
const expenses = await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 ORDER BY "date" DESC, "created_at" DESC`,
[
userId,
],
);
return expenses.map((expense) => ({
description: expense.description,
budget: expense.budget,
date: expense.date,
is_recurring: expense.is_recurring,
// Numeric values come as strings, so we need to convert them to numbers
cost: Number(expense.cost),
extra: {},
}));
}
static async getById(userId: string, id: string) {
const expense = (await db.query<Expense>(
sql`SELECT * FROM "bewcloud_expenses" WHERE "user_id" = $1 AND "id" = $2`,
[userId, id],
))[0];
if (!expense) {
return null;
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...expense,
cost: Number(expense.cost),
};
}
static async create(
userId: string,
cost: number,
description: string,
budget: string,
date: string,
is_recurring: boolean,
{ skipRecalculation = false, skipBudgetMatching = false, skipBudgetCreation = false }: {
skipRecalculation?: boolean;
skipBudgetMatching?: boolean;
skipBudgetCreation?: boolean;
} = {},
) {
const extra: Expense['extra'] = {};
if (!budget.trim()) {
budget = 'Misc';
}
// Match budget to an existing expense "by default"
if (!skipBudgetMatching && budget === 'Misc') {
const existingExpense = await this.getByName(userId, description);
if (existingExpense) {
budget = existingExpense.budget;
}
}
if (!skipBudgetCreation) {
const existingBudgetInMonth = await BudgetModel.getByName(userId, date.substring(0, 7), budget);
if (!existingBudgetInMonth) {
await BudgetModel.create(userId, budget, date.substring(0, 7), 100);
}
}
const newExpense = (await db.query<Expense>(
sql`INSERT INTO "bewcloud_expenses" (
"user_id",
"cost",
"description",
"budget",
"date",
"is_recurring",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *`,
[
userId,
cost,
description,
budget,
date,
is_recurring,
JSON.stringify(extra),
],
))[0];
if (!skipRecalculation) {
await recalculateMonthBudgets(userId, date.substring(0, 7));
}
// Numeric values come as strings, so we need to convert them to numbers
return {
...newExpense,
cost: Number(newExpense.cost),
};
}
static async update(expense: Expense) {
const existingBudgetInMonth = await BudgetModel.getByName(
expense.user_id,
expense.date.substring(0, 7),
expense.budget,
);
if (!existingBudgetInMonth) {
await BudgetModel.create(expense.user_id, expense.budget, expense.date.substring(0, 7), 100);
}
await db.query(
sql`UPDATE "bewcloud_expenses" SET
"cost" = $2,
"description" = $3,
"budget" = $4,
"date" = $5,
"is_recurring" = $6,
"extra" = $7
WHERE "id" = $1`,
[
expense.id,
expense.cost,
expense.description,
expense.budget,
expense.date,
expense.is_recurring,
JSON.stringify(expense.extra),
],
);
await recalculateMonthBudgets(expense.user_id, expense.date.substring(0, 7));
}
static async delete(userId: string, id: string) {
const expense = await this.getById(userId, id);
await db.query(
sql`DELETE FROM "bewcloud_expenses" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await recalculateMonthBudgets(userId, expense!.date.substring(0, 7));
}
}
export async function deleteAllBudgetsAndExpenses(userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_expenses" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_budgets" WHERE "user_id" = $1`,
[
userId,
],
);
}
export async function generateMonthlyBudgetsAndExpenses(userId: string, month: string) {
const lock = new Locker(`expenses:${userId}:${month}`);
await lock.acquire();
let addedBudgetsCount = 0;
let addedExpensesCount = 0;
try {
// Confirm there are no budgets or expenses for this month
const monthBudgets = await BudgetModel.list(userId, month, { skipRecalculation: true });
const monthExpenses = await ExpenseModel.list(userId, month);
if (monthBudgets.length > 0 || monthExpenses.length > 0) {
throw new Error('Budgets and expenses already exist for this month!');
}
// Get the previous month's budgets, to copy over
const previousMonthDate = new Date(month);
previousMonthDate.setMonth(previousMonthDate.getMonth() - 1);
const previousMonth = previousMonthDate.toISOString().substring(0, 7);
const budgets = await BudgetModel.list(userId, previousMonth, { skipRecalculation: true });
for (const budget of budgets) {
await BudgetModel.create(userId, budget.name, month, budget.value);
addedBudgetsCount++;
}
// Get the recurring expenses for the previous month, to copy over
const recurringExpenses = (await ExpenseModel.list(userId, previousMonth)).filter((expense) =>
expense.is_recurring
);
for (const expense of recurringExpenses) {
await ExpenseModel.create(
userId,
expense.cost,
expense.description,
expense.budget,
expense.date.replace(previousMonth, month),
expense.is_recurring,
{ skipRecalculation: true },
);
addedExpensesCount++;
}
console.info(`Added ${addedBudgetsCount} new budgets and ${addedExpensesCount} new expenses for ${month}`);
lock.release();
} catch (error) {
lock.release();
throw error;
}
}
export async function recalculateMonthBudgets(userId: string, month: string) {
const lock = new Locker(`expenses:${userId}:${month}`);
await lock.acquire();
try {
const budgets = await BudgetModel.list(userId, month, { skipRecalculation: true });
const expenses = await ExpenseModel.list(userId, month);
// Calculate total expenses for each budget
const budgetExpenses = new Map<string, number>();
for (const expense of expenses) {
const currentTotal = Number(budgetExpenses.get(expense.budget) || 0);
budgetExpenses.set(expense.budget, Number(currentTotal + expense.cost));
}
// Update each budget with new calculations
for (const budget of budgets) {
const usedValue = Number(budgetExpenses.get(budget.name) || 0);
const availableValue = Number(budget.value - usedValue);
const updatedBudget: Budget = {
...budget,
extra: {
...budget.extra,
usedValue,
availableValue,
},
};
if (budget.extra.usedValue !== usedValue || budget.extra.availableValue !== availableValue) {
await BudgetModel.update(updatedBudget, { skipRecalculation: true });
}
}
lock.release();
} catch (error) {
lock.release();
throw error;
}
}

538
lib/models/files.ts Normal file
View File

@@ -0,0 +1,538 @@
import { join } from 'std/path/join.ts';
import { resolve } from 'std/path/resolve.ts';
import { lookup } from 'mrmime';
import { getFilesRootPath } from '/lib/config.ts';
import { Directory, DirectoryFile } from '/lib/types.ts';
import { sortDirectoriesByName, sortEntriesByName, sortFilesByName, TRASH_PATH } from '/lib/utils/files.ts';
export class DirectoryModel {
static async list(userId: string, path: string): Promise<Directory[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
const directories: Directory[] = [];
const directoryEntries = (await getPathEntries(userId, path)).filter((entry) =>
entry.isDirectory || entry.isSymlink
);
for (const entry of directoryEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const directory: Directory = {
user_id: userId,
parent_path: path,
directory_name: entry.name,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
directories.push(directory);
}
directories.sort(sortDirectoriesByName);
return directories;
}
static async create(userId: string, path: string, name: string): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
await Deno.mkdir(join(rootPath, name), { recursive: true });
} catch (error) {
console.error(error);
return false;
}
return true;
}
static async rename(
userId: string,
oldPath: string,
newPath: string,
oldName: string,
newName: string,
): Promise<boolean> {
return await renameDirectoryOrFile(userId, oldPath, newPath, oldName, newName);
}
static async delete(userId: string, path: string, name: string): Promise<boolean> {
return await deleteDirectoryOrFile(userId, path, name);
}
static async searchNames(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; directories: Directory[] }> {
const rootPath = join(getFilesRootPath(), userId);
const directories: Directory[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`find`, {
args: [
`.`, // proper cwd is sent below
`-type`,
`d,l`, // directories and symbolic links
`-iname`,
`*${searchTerm}*`,
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code !== 0) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "find"`);
}
const output = new TextDecoder().decode(stdout);
const matchingDirectories = output.split('\n').map((directoryPath) => directoryPath.trim()).filter(Boolean);
for (const relativeDirectoryPath of matchingDirectories) {
const stat = await Deno.stat(join(rootPath, relativeDirectoryPath));
let parentPath = `/${relativeDirectoryPath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const directoryName = relativeDirectoryPath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const directory: Directory = {
user_id: userId,
parent_path: parentPath,
directory_name: directoryName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
directories.push(directory);
}
return { success: true, directories };
} catch (error) {
console.error(error);
}
return { success: false, directories };
}
}
export class FileModel {
static async list(userId: string, path: string): Promise<DirectoryFile[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
const files: DirectoryFile[] = [];
const fileEntries = (await getPathEntries(userId, path)).filter((entry) => entry.isFile);
for (const entry of fileEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const file: DirectoryFile = {
user_id: userId,
parent_path: path,
file_name: entry.name,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
files.sort(sortFilesByName);
return files;
}
static async create(
userId: string,
path: string,
name: string,
contents: string | ArrayBuffer,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
// Ensure the directory exist, if being requested
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(rootPath, { recursive: true });
}
}
if (typeof contents === 'string') {
await Deno.writeTextFile(join(rootPath, name), contents, { append: false, createNew: true });
} else {
await Deno.writeFile(join(rootPath, name), new Uint8Array(contents), { append: false, createNew: true });
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
static async update(
userId: string,
path: string,
name: string,
contents: string,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
await Deno.writeTextFile(join(rootPath, name), contents, { append: false, createNew: false });
} catch (error) {
console.error(error);
return false;
}
return true;
}
static async get(
userId: string,
path: string,
name?: string,
): Promise<{ success: boolean; contents?: Uint8Array; contentType?: string; byteSize?: number }> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name || ''));
const rootPath = join(getFilesRootPath(), userId, path);
try {
const stat = await Deno.stat(join(rootPath, name || ''));
if (stat) {
const contents = await Deno.readFile(join(rootPath, name || ''));
const extension = (name || path).split('.').slice(-1).join('').toLowerCase();
const contentType = lookup(extension) || 'application/octet-stream';
return {
success: true,
contents,
contentType,
byteSize: stat.size,
};
}
} catch (error) {
console.error(error);
}
return {
success: false,
};
}
static async rename(
userId: string,
oldPath: string,
newPath: string,
oldName: string,
newName: string,
): Promise<boolean> {
return await renameDirectoryOrFile(userId, oldPath, newPath, oldName, newName);
}
static async delete(userId: string, path: string, name: string): Promise<boolean> {
return await deleteDirectoryOrFile(userId, path, name);
}
static async searchNames(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; files: DirectoryFile[] }> {
const rootPath = join(getFilesRootPath(), userId);
const files: DirectoryFile[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`find`, {
args: [
`.`, // proper cwd is sent below
`-type`,
`f`,
`-iname`,
`*${searchTerm}*`,
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code !== 0) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "find"`);
}
const output = new TextDecoder().decode(stdout);
const matchingFiles = output.split('\n').map((filePath) => filePath.trim()).filter(Boolean);
for (const relativeFilePath of matchingFiles) {
const stat = await Deno.stat(join(rootPath, relativeFilePath));
let parentPath = `/${relativeFilePath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const fileName = relativeFilePath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const file: DirectoryFile = {
user_id: userId,
parent_path: parentPath,
file_name: fileName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
return { success: true, files };
} catch (error) {
console.error(error);
}
return { success: false, files };
}
static async searchContents(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; files: DirectoryFile[] }> {
const rootPath = join(getFilesRootPath(), userId);
const files: DirectoryFile[] = [];
try {
const controller = new AbortController();
const commandTimeout = setTimeout(() => controller.abort(), 10_000);
const command = new Deno.Command(`grep`, {
args: [
`-rHisl`,
`${searchTerm}`,
`.`, // proper cwd is sent below
],
cwd: rootPath,
signal: controller.signal,
});
const { code, stdout, stderr } = await command.output();
if (commandTimeout) {
clearTimeout(commandTimeout);
}
if (code > 1) {
if (stderr) {
throw new Error(new TextDecoder().decode(stderr));
}
throw new Error(`Unknown error running "grep"`);
}
const output = new TextDecoder().decode(stdout);
const matchingFiles = output.split('\n').map((filePath) => filePath.trim()).filter(Boolean);
for (const relativeFilePath of matchingFiles) {
const stat = await Deno.stat(join(rootPath, relativeFilePath));
let parentPath = `/${relativeFilePath.replace('./', '/').split('/').slice(0, -1).join('')}/`;
const fileName = relativeFilePath.split('/').pop()!;
if (parentPath === '//') {
parentPath = '/';
}
const file: DirectoryFile = {
user_id: userId,
parent_path: parentPath,
file_name: fileName,
has_write_access: true,
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
return { success: true, files };
} catch (error) {
console.error(error);
}
return { success: false, files };
}
}
/**
* Ensures the user path is valid and securely accessible (meaning it's not trying to access files outside of the user's root directory).
* Does not check if the path exists.
*
* @param userId - The user ID
* @param path - The relative path (user-provided) to check
*/
export function ensureUserPathIsValidAndSecurelyAccessible(userId: string, path: string): void {
const userRootPath = join(getFilesRootPath(), userId, '/');
const fullPath = join(userRootPath, path);
const resolvedFullPath = `${resolve(fullPath)}/`;
if (!resolvedFullPath.startsWith(userRootPath)) {
throw new Error('Invalid file path');
}
}
async function getPathEntries(userId: string, path: string): Promise<Deno.DirEntry[]> {
ensureUserPathIsValidAndSecurelyAccessible(userId, path);
const rootPath = join(getFilesRootPath(), userId, path);
// Ensure the user directory exists
if (path === '/') {
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(join(rootPath, TRASH_PATH), { recursive: true });
}
}
}
// Ensure the Notes or Photos directories exist, if being requested
if (path === '/Notes/' || path === '/Photos/') {
try {
await Deno.stat(rootPath);
} catch (error) {
if ((error as Error).toString().includes('NotFound')) {
await Deno.mkdir(rootPath, { recursive: true });
}
}
}
const entries: Deno.DirEntry[] = [];
for await (const dirEntry of Deno.readDir(rootPath)) {
entries.push(dirEntry);
}
entries.sort(sortEntriesByName);
return entries;
}
async function renameDirectoryOrFile(
userId: string,
oldPath: string,
newPath: string,
oldName: string,
newName: string,
): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(oldPath, oldName));
ensureUserPathIsValidAndSecurelyAccessible(userId, join(newPath, newName));
const oldRootPath = join(getFilesRootPath(), userId, oldPath);
const newRootPath = join(getFilesRootPath(), userId, newPath);
try {
await Deno.rename(join(oldRootPath, oldName), join(newRootPath, newName));
} catch (error) {
console.error(error);
return false;
}
return true;
}
async function deleteDirectoryOrFile(userId: string, path: string, name: string): Promise<boolean> {
ensureUserPathIsValidAndSecurelyAccessible(userId, join(path, name));
const rootPath = join(getFilesRootPath(), userId, path);
try {
if (path.startsWith(TRASH_PATH)) {
await Deno.remove(join(rootPath, name), { recursive: true });
} else {
const trashPath = join(getFilesRootPath(), userId, TRASH_PATH);
await Deno.rename(join(rootPath, name), join(trashPath, name));
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function searchFilesAndDirectories(
userId: string,
searchTerm: string,
): Promise<{ success: boolean; directories: Directory[]; files: DirectoryFile[] }> {
const directoryNamesResult = await DirectoryModel.searchNames(userId, searchTerm);
const fileNamesResult = await FileModel.searchNames(userId, searchTerm);
const fileContentsResult = await FileModel.searchContents(userId, searchTerm);
const success = directoryNamesResult.success && fileNamesResult.success && fileContentsResult.success;
const directories = [...directoryNamesResult.directories];
directories.sort(sortDirectoriesByName);
const files = [...fileNamesResult.files, ...fileContentsResult.files];
files.sort(sortFilesByName);
return {
success,
directories,
files,
};
}

315
lib/models/news.ts Normal file
View File

@@ -0,0 +1,315 @@
import { Feed } from 'https://deno.land/x/rss@1.0.0/mod.ts';
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { NewsFeed, NewsFeedArticle } from '/lib/types.ts';
import {
findFeedInUrl,
getArticleUrl,
getFeedInfo,
JsonFeed,
parseTextFromHtml,
parseUrl,
parseUrlAsGooglebot,
parseUrlWithProxy,
} from '/lib/feed.ts';
const db = new Database();
export class FeedModel {
static async list(userId: string) {
const newsFeeds = await db.query<NewsFeed>(sql`SELECT * FROM "bewcloud_news_feeds" WHERE "user_id" = $1`, [
userId,
]);
return newsFeeds;
}
static async get(id: string, userId: string) {
const newsFeeds = await db.query<NewsFeed>(
sql`SELECT * FROM "bewcloud_news_feeds" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return newsFeeds[0];
}
static async create(userId: string, feedUrl: string) {
const extra: NewsFeed['extra'] = {};
const newNewsFeed = (await db.query<NewsFeed>(
sql`INSERT INTO "bewcloud_news_feeds" (
"user_id",
"feed_url",
"extra"
) VALUES ($1, $2, $3)
RETURNING *`,
[
userId,
feedUrl,
JSON.stringify(extra),
],
))[0];
return newNewsFeed;
}
static async update(newsFeed: NewsFeed) {
await db.query(
sql`UPDATE "bewcloud_news_feeds" SET
"feed_url" = $2,
"last_crawled_at" = $3,
"extra" = $4
WHERE "id" = $1`,
[
newsFeed.id,
newsFeed.feed_url,
newsFeed.last_crawled_at,
JSON.stringify(newsFeed.extra),
],
);
}
static async delete(id: string, userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_news_feed_articles" WHERE "feed_id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feeds" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
}
static async crawl(newsFeed: NewsFeed) {
type FeedArticle = Feed['entries'][number];
type JsonFeedArticle = JsonFeed['items'][number];
const MAX_ARTICLES_CRAWLED_PER_RUN = 10;
const lock = new Locker(`feeds:${newsFeed.id}`);
await lock.acquire();
try {
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
const feedUrl = await findFeedInUrl(newsFeed.feed_url);
if (!feedUrl) {
throw new Error(
`Invalid URL for feed: "${feedUrl}"`,
);
}
if (feedUrl !== newsFeed.feed_url) {
newsFeed.feed_url = feedUrl;
}
const feedInfo = await getFeedInfo(newsFeed.feed_url);
newsFeed.extra.title = feedInfo.title;
newsFeed.extra.feed_type = feedInfo.feed_type;
newsFeed.extra.crawl_type = feedInfo.crawl_type;
}
const feedArticles = await fetchNewsArticles(newsFeed);
const articles: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>[] = [];
for (const feedArticle of feedArticles) {
// Don't add too many articles per run
if (articles.length >= MAX_ARTICLES_CRAWLED_PER_RUN) {
continue;
}
const url = (feedArticle as JsonFeedArticle).url || getArticleUrl((feedArticle as FeedArticle).links) ||
feedArticle.id;
const articleIsoDate = (feedArticle as JsonFeedArticle).date_published ||
(feedArticle as FeedArticle).published?.toISOString() || (feedArticle as JsonFeedArticle).date_modified ||
(feedArticle as FeedArticle).updated?.toISOString();
const articleDate = articleIsoDate ? new Date(articleIsoDate) : new Date();
const summary = await parseTextFromHtml(
(feedArticle as FeedArticle).description?.value || (feedArticle as FeedArticle).content?.value ||
(feedArticle as JsonFeedArticle).content_text || (feedArticle as JsonFeedArticle).content_html ||
(feedArticle as JsonFeedArticle).summary || '',
);
if (url) {
articles.push({
article_title: (feedArticle as FeedArticle).title?.value || (feedArticle as JsonFeedArticle).title ||
url.replace('http://', '').replace('https://', ''),
article_url: url,
article_summary: summary,
article_date: articleDate,
});
}
}
const existingArticles = await ArticleModel.listByFeedId(newsFeed.id);
const existingArticleUrls = new Set<string>(existingArticles.map((article) => article.article_url));
const previousLatestArticleUrl = existingArticles[0]?.article_url;
let seenPreviousLatestArticleUrl = false;
let addedArticlesCount = 0;
for (const article of articles) {
// Stop looking after seeing the previous latest article
if (article.article_url === previousLatestArticleUrl) {
seenPreviousLatestArticleUrl = true;
}
if (!seenPreviousLatestArticleUrl && !existingArticleUrls.has(article.article_url)) {
try {
await ArticleModel.create(newsFeed.user_id, newsFeed.id, article);
++addedArticlesCount;
} catch (error) {
console.error(error);
console.error(`Failed to add new article: "${article.article_url}"`);
}
}
}
console.info('Added', addedArticlesCount, 'new articles');
newsFeed.last_crawled_at = new Date();
await this.update(newsFeed);
lock.release();
} catch (error) {
lock.release();
throw error;
}
}
}
export class ArticleModel {
static async list(userId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "user_id" = $1 ORDER BY "article_date" DESC`,
[
userId,
],
);
return articles;
}
static async listByFeedId(feedId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "feed_id" = $1 ORDER BY "article_date" DESC`,
[
feedId,
],
);
return articles;
}
static async get(id: string, userId: string) {
const articles = await db.query<NewsFeedArticle>(
sql`SELECT * FROM "bewcloud_news_feed_articles" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return articles[0];
}
static async create(
userId: string,
feedId: string,
article: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>,
) {
const extra: NewsFeedArticle['extra'] = {};
const newNewsArticle = (await db.query<NewsFeedArticle>(
sql`INSERT INTO "bewcloud_news_feed_articles" (
"user_id",
"feed_id",
"article_url",
"article_title",
"article_summary",
"article_date",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *`,
[
userId,
feedId,
article.article_url,
article.article_title,
article.article_summary,
article.article_date,
JSON.stringify(extra),
],
))[0];
return newNewsArticle;
}
static async update(article: NewsFeedArticle) {
await db.query(
sql`UPDATE "bewcloud_news_feed_articles" SET
"is_read" = $2,
"extra" = $3
WHERE "id" = $1`,
[
article.id,
article.is_read,
JSON.stringify(article.extra),
],
);
}
static async markAllRead(userId: string) {
await db.query(
sql`UPDATE "bewcloud_news_feed_articles" SET
"is_read" = TRUE
WHERE "user_id" = $1`,
[
userId,
],
);
}
}
async function fetchNewsArticles(newsFeed: NewsFeed): Promise<Feed['entries'] | JsonFeed['items']> {
try {
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
throw new Error('Invalid News Feed!');
}
let feed: JsonFeed | Feed | null = null;
if (newsFeed.extra.crawl_type === 'direct') {
feed = await parseUrl(newsFeed.feed_url);
} else if (newsFeed.extra.crawl_type === 'googlebot') {
feed = await parseUrlAsGooglebot(newsFeed.feed_url);
} else if (newsFeed.extra.crawl_type === 'proxy') {
feed = await parseUrlWithProxy(newsFeed.feed_url);
}
return (feed as Feed)?.entries || (feed as JsonFeed)?.items || [];
} catch (error) {
console.error('Failed parsing feed to get articles', newsFeed.feed_url);
console.error(error);
}
return [];
}

290
lib/models/user.ts Normal file
View File

@@ -0,0 +1,290 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import { User, UserSession, VerificationCode } from '/lib/types.ts';
import { generateRandomCode } from '/lib/utils/misc.ts';
import { isEmailEnabled, isForeverSignupEnabled } from '/lib/config.ts';
const db = new Database();
export class UserModel {
static async isThereAnAdmin() {
const user = (await db.query<User>(
sql`SELECT * FROM "bewcloud_users" WHERE ("extra" ->> 'is_admin')::boolean IS TRUE LIMIT 1`,
))[
0
];
return Boolean(user);
}
static async getByEmail(email: string) {
const lowercaseEmail = email.toLowerCase().trim();
const user = (await db.query<User>(sql`SELECT * FROM "bewcloud_users" WHERE "email" = $1 LIMIT 1`, [
lowercaseEmail,
]))[0];
return user;
}
static async getById(id: string) {
const user = (await db.query<User>(sql`SELECT * FROM "bewcloud_users" WHERE "id" = $1 LIMIT 1`, [
id,
]))[0];
return user;
}
static async create(email: User['email'], hashedPassword: User['hashed_password']) {
const trialDays = isForeverSignupEnabled() ? 36_525 : 30;
const now = new Date();
const trialEndDate = new Date(new Date().setUTCDate(new Date().getUTCDate() + trialDays));
const subscription: User['subscription'] = {
external: {},
expires_at: trialEndDate.toISOString(),
updated_at: now.toISOString(),
};
const extra: User['extra'] = { is_email_verified: isEmailEnabled() ? false : true };
// First signup will be an admin "forever"
if (!(await this.isThereAnAdmin())) {
extra.is_admin = true;
subscription.expires_at = new Date('2100-12-31').toISOString();
}
const newUser = (await db.query<User>(
sql`INSERT INTO "bewcloud_users" (
"email",
"subscription",
"status",
"hashed_password",
"extra"
) VALUES ($1, $2, $3, $4, $5)
RETURNING *`,
[
email,
JSON.stringify(subscription),
extra.is_admin || isForeverSignupEnabled() ? 'active' : 'trial',
hashedPassword,
JSON.stringify(extra),
],
))[0];
return newUser;
}
static async update(user: User) {
await db.query(
sql`UPDATE "bewcloud_users" SET
"email" = $2,
"subscription" = $3,
"status" = $4,
"hashed_password" = $5,
"extra" = $6
WHERE "id" = $1`,
[
user.id,
user.email,
JSON.stringify(user.subscription),
user.status,
user.hashed_password,
JSON.stringify(user.extra),
],
);
}
static async delete(userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_user_sessions" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_verification_codes" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feed_articles" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_news_feeds" WHERE "user_id" = $1`,
[
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_users" WHERE "id" = $1`,
[
userId,
],
);
}
}
export class UserSessionModel {
static async getById(id: string) {
const session = (await db.query<UserSession>(
sql`SELECT * FROM "bewcloud_user_sessions" WHERE "id" = $1 AND "expires_at" > now() LIMIT 1`,
[
id,
],
))[0];
return session;
}
static async create(user: User, isShortLived = false) {
const oneMonthFromToday = new Date(new Date().setUTCMonth(new Date().getUTCMonth() + 1));
const oneWeekFromToday = new Date(new Date().setUTCDate(new Date().getUTCDate() + 7));
const newSession: Omit<UserSession, 'id' | 'created_at'> = {
user_id: user.id,
expires_at: isShortLived ? oneWeekFromToday : oneMonthFromToday,
last_seen_at: new Date(),
};
const newUserSessionResult = (await db.query<UserSession>(
sql`INSERT INTO "bewcloud_user_sessions" (
"user_id",
"expires_at",
"last_seen_at"
) VALUES ($1, $2, $3)
RETURNING *`,
[
newSession.user_id,
newSession.expires_at,
newSession.last_seen_at,
],
))[0];
return newUserSessionResult;
}
static async update(session: UserSession) {
await db.query(
sql`UPDATE "bewcloud_user_sessions" SET
"expires_at" = $2,
"last_seen_at" = $3
WHERE "id" = $1`,
[
session.id,
session.expires_at,
session.last_seen_at,
],
);
}
static async delete(sessionId: string) {
await db.query(
sql`DELETE FROM "bewcloud_user_sessions" WHERE "id" = $1`,
[
sessionId,
],
);
}
}
export async function validateUserAndSession(userId: string, sessionId: string) {
const user = await UserModel.getById(userId);
if (!user) {
throw new Error('Not Found');
}
const session = await UserSessionModel.getById(sessionId);
if (!session || session.user_id !== user.id) {
throw new Error('Not Found');
}
session.last_seen_at = new Date();
await UserSessionModel.update(session);
return { user, session };
}
export class VerificationCodeModel {
static async create(
user: User,
verificationId: string,
type: VerificationCode['verification']['type'],
) {
const inThirtyMinutes = new Date(new Date().setUTCMinutes(new Date().getUTCMinutes() + 30));
const code = generateRandomCode();
const newVerificationCode: Omit<VerificationCode, 'id' | 'created_at'> = {
user_id: user.id,
code,
expires_at: inThirtyMinutes,
verification: {
id: verificationId,
type,
},
};
await db.query(
sql`INSERT INTO "bewcloud_verification_codes" (
"user_id",
"code",
"expires_at",
"verification"
) VALUES ($1, $2, $3, $4)
RETURNING "id"`,
[
newVerificationCode.user_id,
newVerificationCode.code,
newVerificationCode.expires_at,
JSON.stringify(newVerificationCode.verification),
],
);
return code;
}
static async validate(
user: User,
verificationId: string,
code: string,
type: VerificationCode['verification']['type'],
) {
const verificationCode = (await db.query<VerificationCode>(
sql`SELECT * FROM "bewcloud_verification_codes"
WHERE "user_id" = $1 AND
"code" = $2 AND
"verification" ->> 'type' = $3 AND
"verification" ->> 'id' = $4 AND
"expires_at" > now()
LIMIT 1`,
[
user.id,
code,
type,
verificationId,
],
))[0];
if (verificationCode) {
await db.query(
sql`DELETE FROM "bewcloud_verification_codes" WHERE "id" = $1`,
[
verificationCode.id,
],
);
} else {
throw new Error('Not Found');
}
}
}